Skip to content

Commit 9f4d4bb

Browse files
jenilsavani9jenilsavani9
committed
feat: Implement asynchronous document generation with progress tracking and Ollama provider support
1 parent d77ca2c commit 9f4d4bb

22 files changed

Lines changed: 1978 additions & 550 deletions

File tree

‎apps/web/app/api/projects/[id]/regenerate/route.ts‎

Lines changed: 51 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,21 @@ import { getServerSession } from "next-auth/next";
33
import { authOptions } from "@/app/api/auth/[...nextauth]/authoptions";
44
import { prisma } from "@/lib/prisma";
55
import { getValidAccessTokenFromInstallation } from "@/lib/github-token";
6-
import { Octokit } from "octokit";
76

87
const ARC42GEN_API_URL = process.env.ARC42GEN_API_URL || "http://localhost:8001";
98

9+
function log(msg: string) {
10+
console.log(`[regenerate] ${new Date().toISOString().slice(11, 19)} ${msg}`);
11+
}
12+
13+
function logError(msg: string, err?: any) {
14+
console.error(`[regenerate] ${new Date().toISOString().slice(11, 19)} ${msg}`);
15+
if (err) console.error(err);
16+
}
17+
1018
/**
1119
* POST /api/projects/[id]/regenerate
12-
* Generate docs and commit to the docs repo
20+
* Submit a doc generation job (returns immediately with job_id)
1321
*/
1422
export async function POST(
1523
req: NextRequest,
@@ -23,7 +31,6 @@ export async function POST(
2331

2432
const { id: projectId } = await params;
2533

26-
// Get project with user's GitHub installation
2734
const project = await prisma.project.findFirst({
2835
where: {
2936
id: projectId,
@@ -49,91 +56,77 @@ export async function POST(
4956
);
5057
}
5158

52-
// Get a valid access token (will auto-refresh if expired)
59+
// Get a valid access token
60+
log(`Getting access token for installation...`);
5361
const tokenResult = await getValidAccessTokenFromInstallation(project.user.gitHubInstallation);
5462
if (tokenResult.error) {
63+
logError(`Token error: ${tokenResult.error}`);
5564
return NextResponse.json(
5665
{ error: tokenResult.error },
5766
{ status: 400 }
5867
);
5968
}
6069
const accessToken = tokenResult.accessToken!;
61-
62-
// Update status to running
63-
await prisma.project.update({
64-
where: { id: projectId },
65-
data: { docGenStatus: "running" },
66-
});
67-
68-
console.log(`Regenerating docs for ${project.name}...`);
69-
console.log(`Source: ${project.sourceRepoUrl}`);
70-
console.log(`Docs repo: ${project.githubOwner}/${project.repoName}`);
71-
72-
// Call arc42gen API with the token for private repos
73-
const genResponse = await fetch(`${ARC42GEN_API_URL}/generate`, {
70+
log(`Token obtained successfully`);
71+
72+
log(`========================================`);
73+
log(`Submitting doc generation job`);
74+
log(` Project: ${project.name}`);
75+
log(` Source: ${project.sourceRepoUrl}`);
76+
log(` Docs repo: ${project.githubOwner}/${project.repoName}`);
77+
log(` API URL: ${ARC42GEN_API_URL}`);
78+
log(`========================================`);
79+
80+
// Submit job to arc42gen API (returns instantly)
81+
const jobResponse = await fetch(`${ARC42GEN_API_URL}/jobs`, {
7482
method: "POST",
7583
headers: { "Content-Type": "application/json" },
7684
body: JSON.stringify({
7785
source_repo_url: project.sourceRepoUrl,
7886
branch: project.sourceRepoBranch || "main",
7987
github_token: accessToken,
88+
owner: project.githubOwner,
89+
repo_name: project.repoName,
90+
source_owner: project.sourceRepoOwner,
91+
source_name: project.sourceRepoName,
92+
project_id: projectId,
8093
}),
94+
signal: AbortSignal.timeout(10_000), // 10s — should be instant
8195
});
8296

83-
const genResult = await genResponse.json();
84-
console.log("Arc42gen result:", genResult);
85-
86-
if (!genResult.success) {
87-
await prisma.project.update({
88-
where: { id: projectId },
89-
data: { docGenStatus: "failed" },
90-
});
91-
return NextResponse.json(
92-
{ error: genResult.error || "Doc generation failed" },
93-
{ status: 500 }
94-
);
95-
}
96-
97-
if (Object.keys(genResult.files).length === 0) {
98-
await prisma.project.update({
99-
where: { id: projectId },
100-
data: { docGenStatus: "failed" },
101-
});
97+
if (!jobResponse.ok) {
98+
const errText = await jobResponse.text();
99+
logError(`Job submission failed: ${jobResponse.status} ${errText}`);
102100
return NextResponse.json(
103-
{ error: "No documentation files were generated" },
101+
{ error: "Failed to submit generation job" },
104102
{ status: 500 }
105103
);
106104
}
107105

108-
// Commit files to docs repo
109-
await commitFilesToRepo({
110-
accessToken,
111-
owner: project.githubOwner!,
112-
repo: project.repoName,
113-
files: genResult.files,
114-
message: `docs: regenerate from ${project.sourceRepoOwner}/${project.sourceRepoName}`,
115-
});
106+
const jobResult = await jobResponse.json();
107+
log(`Job submitted: ${jobResult.job_id}`);
116108

117-
// Success!
109+
// Store job_id and set status to running
118110
await prisma.project.update({
119111
where: { id: projectId },
120112
data: {
121-
docGenStatus: "success",
122-
lastDocGenAt: new Date(),
113+
docGenStatus: "running",
114+
docGenJobId: jobResult.job_id,
123115
},
124116
});
125117

126-
console.log(`Docs regenerated successfully for ${project.name}`);
127-
128-
return NextResponse.json({
129-
success: true,
130-
message: "Documentation regenerated and committed",
131-
filesCommitted: Object.keys(genResult.files).length,
132-
});
118+
return NextResponse.json(
119+
{
120+
success: true,
121+
job_id: jobResult.job_id,
122+
message: "Documentation generation job submitted",
123+
},
124+
{ status: 202 }
125+
);
133126

134127
} catch (error: any) {
135-
console.error("Regenerate error:", error);
136-
128+
logError(`Failed to submit job: ${error.message}`, error);
129+
137130
// Try to update status to failed
138131
try {
139132
const { id } = await params;
@@ -144,90 +137,8 @@ export async function POST(
144137
} catch {}
145138

146139
return NextResponse.json(
147-
{ error: error.message || "Failed to regenerate docs" },
140+
{ error: error.message || "Failed to submit generation job" },
148141
{ status: 500 }
149142
);
150143
}
151144
}
152-
153-
/**
154-
* Commit files to a GitHub repository
155-
*/
156-
async function commitFilesToRepo(options: {
157-
accessToken: string;
158-
owner: string;
159-
repo: string;
160-
files: Record<string, string>;
161-
message: string;
162-
}) {
163-
const { accessToken, owner, repo, files, message } = options;
164-
const octokit = new Octokit({ auth: accessToken });
165-
166-
console.log(`Committing ${Object.keys(files).length} files to ${owner}/${repo}`);
167-
168-
// Get default branch
169-
const { data: repoData } = await octokit.request("GET /repos/{owner}/{repo}", {
170-
owner, repo,
171-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
172-
});
173-
const branch = repoData.default_branch;
174-
175-
// Get latest commit
176-
const { data: ref } = await octokit.request("GET /repos/{owner}/{repo}/git/ref/{ref}", {
177-
owner, repo, ref: `heads/${branch}`,
178-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
179-
});
180-
const latestCommitSha = ref.object.sha;
181-
182-
// Get commit tree
183-
const { data: commit } = await octokit.request("GET /repos/{owner}/{repo}/git/commits/{commit_sha}", {
184-
owner, repo, commit_sha: latestCommitSha,
185-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
186-
});
187-
188-
// Create blobs for each file
189-
const blobs = await Promise.all(
190-
Object.entries(files).map(async ([path, content]) => {
191-
const { data: blob } = await octokit.request("POST /repos/{owner}/{repo}/git/blobs", {
192-
owner, repo,
193-
content: Buffer.from(content).toString("base64"),
194-
encoding: "base64",
195-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
196-
});
197-
// Put files in docs/ folder
198-
return { path: `docs/${path}`, sha: blob.sha };
199-
})
200-
);
201-
202-
// Create tree
203-
const { data: newTree } = await octokit.request("POST /repos/{owner}/{repo}/git/trees", {
204-
owner, repo,
205-
base_tree: commit.tree.sha,
206-
tree: blobs.map((b) => ({
207-
path: b.path,
208-
mode: "100644" as const,
209-
type: "blob" as const,
210-
sha: b.sha,
211-
})),
212-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
213-
});
214-
215-
// Create commit
216-
const { data: newCommit } = await octokit.request("POST /repos/{owner}/{repo}/git/commits", {
217-
owner, repo,
218-
message,
219-
tree: newTree.sha,
220-
parents: [latestCommitSha],
221-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
222-
});
223-
224-
// Update branch
225-
await octokit.request("PATCH /repos/{owner}/{repo}/git/refs/{ref}", {
226-
owner, repo,
227-
ref: `heads/${branch}`,
228-
sha: newCommit.sha,
229-
headers: { "X-GitHub-Api-Version": "2022-11-28" },
230-
});
231-
232-
console.log(`Committed to ${owner}/${repo}@${branch}`);
233-
}

‎apps/web/app/api/projects/[id]/regenerate/status/route.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
import { NextRequest, NextResponse } from "next/server";
2+
import { getServerSession } from "next-auth/next";
3+
import { authOptions } from "@/app/api/auth/[...nextauth]/authoptions";
4+
import { prisma } from "@/lib/prisma";
5+
6+
const ARC42GEN_API_URL = process.env.ARC42GEN_API_URL || "http://localhost:8001";
7+
8+
/**
9+
* GET /api/projects/[id]/regenerate/status
10+
* Poll the status of a doc generation job
11+
*/
12+
export async function GET(
13+
req: NextRequest,
14+
{ params }: { params: Promise<{ id: string }> }
15+
) {
16+
try {
17+
const session = await getServerSession(authOptions);
18+
if (!session?.user?.id) {
19+
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
20+
}
21+
22+
const { id: projectId } = await params;
23+
24+
const project = await prisma.project.findFirst({
25+
where: {
26+
id: projectId,
27+
userId: session.user.id,
28+
},
29+
select: {
30+
docGenStatus: true,
31+
docGenJobId: true,
32+
lastDocGenAt: true,
33+
},
34+
});
35+
36+
if (!project) {
37+
return NextResponse.json({ error: "Project not found" }, { status: 404 });
38+
}
39+
40+
// If no job ID, just return the stored status
41+
if (!project.docGenJobId) {
42+
return NextResponse.json({
43+
status: project.docGenStatus || "idle",
44+
progress: 0,
45+
current_step: null,
46+
error: null,
47+
});
48+
}
49+
50+
// Poll the arc42gen API for job status
51+
const statusResponse = await fetch(
52+
`${ARC42GEN_API_URL}/jobs/${project.docGenJobId}`,
53+
{ signal: AbortSignal.timeout(5_000) }
54+
);
55+
56+
if (!statusResponse.ok) {
57+
return NextResponse.json({
58+
status: project.docGenStatus || "unknown",
59+
progress: 0,
60+
current_step: null,
61+
error: "Could not fetch job status",
62+
});
63+
}
64+
65+
const jobStatus = await statusResponse.json();
66+
67+
// Update project status if job has completed
68+
if (jobStatus.status === "success" && project.docGenStatus !== "success") {
69+
await prisma.project.update({
70+
where: { id: projectId },
71+
data: {
72+
docGenStatus: "success",
73+
lastDocGenAt: new Date(),
74+
},
75+
});
76+
} else if (jobStatus.status === "failed" && project.docGenStatus !== "failed") {
77+
await prisma.project.update({
78+
where: { id: projectId },
79+
data: { docGenStatus: "failed" },
80+
});
81+
}
82+
83+
return NextResponse.json({
84+
status: jobStatus.status,
85+
progress: jobStatus.progress,
86+
current_step: jobStatus.current_step,
87+
error: jobStatus.error,
88+
result: jobStatus.result,
89+
});
90+
91+
} catch (error: any) {
92+
return NextResponse.json(
93+
{ error: error.message || "Failed to get job status" },
94+
{ status: 500 }
95+
);
96+
}
97+
}

0 commit comments

Comments
 (0)