import { env } from "cloudflare:workers"; import { batch, first, run, stmt } from "../db"; import { repoDb, stmt as scopedStmt } from "../scoped-db"; import { registerLocator } from "../scoped-db/locators"; import { scopedId } from "../scoped-db/ids"; import { registerRepoRows } from "../issue-scope"; import { randomId } from "../crypto"; import { allocateIssueNumber, applyLabels, indexIssue } from "../issues"; import { loadRepoById } from "../repos"; import { HttpError } from "../errors"; import { githubApi, installationToken } from "./github-app"; import { assertMirrorPullAllowed, isGithubRepoName } from "./github-sync"; import { loadIntegration, parseConfig } from "./store"; const PAGE_SIZE = 20; const MAX_IMPORT_ITEMS = 2000; const MAX_COMMENTS_PER_ITEM = 30; const MAX_PAGE_BYTES = 2 * 1024 * 1024; const MAX_BODY = 60_000; const IMPORT_LABELS = [ { name: "imported", color: "6e7681" }, { name: "pull-request", color: "8957e5" }, ]; export type ImportJobMessage = { kind: "integration.job"; jobId: string; page: number; }; type GithubIssue = { number: number; title: string; body: string | null; state: "open" | "closed"; html_url: string; comments: number; created_at: string; closed_at: string | null; user: { login: string } | null; pull_request?: { merged_at: string | null }; }; type GithubComment = { body: string | null; created_at: string; user: { login: string } | null; html_url: string; }; type JobRow = { id: string; integration_id: string; status: string; progress: string; }; type Progress = { imported: number; pages: number; done?: boolean }; function clipBody(text: string): string { return text.length > MAX_BODY ? `${text.slice(0, MAX_BODY)}\n\n_(truncated)_` : text; } function attribution(kind: string, login: string | null, url: string, when: string): string { return `> _${kind} [on GitHub](${url}) by @${login ?? "ghost"} on ${when.slice(0, 10)}._\n\n`; } async function updateJob( jobId: string, status: string, progress: Progress, error: string | null = null, ): Promise { await run( "UPDATE integration_jobs SET status = ?, progress = ?, error = ?, updated_at = ? WHERE id = ?", status, JSON.stringify(progress), error, Date.now(), jobId, ); } async function importComments( token: string, githubRepo: string, item: GithubIssue, repoId: string, issueId: string, authorId: string, ) { const fallback = Date.parse(item.created_at) || Date.now(); const comments = item.comments > 0 ? await githubApi( token, `/repos/${githubRepo}/issues/${item.number}/comments?per_page=${MAX_COMMENTS_PER_ITEM}`, MAX_PAGE_BYTES, ) : []; const kept = comments.slice(0, MAX_COMMENTS_PER_ITEM); const ids = kept.map(() => scopedId(repoId)); const statements = kept.map((comment, index) => { const when = Date.parse(comment.created_at) || fallback; return scopedStmt( "INSERT INTO comments (id, issue_id, author_id, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)", ids[index], issueId, authorId, clipBody( attribution("Comment", comment.user?.login ?? null, comment.html_url, comment.created_at) + (comment.body ?? ""), ), when, when, ); }); // A retry after a crash between these two re-imports the comments (the DELETE keeps that from duplicating them). await repoDb(repoId).batch([ scopedStmt("DELETE FROM comments WHERE issue_id = ?", issueId), ...statements, scopedStmt("UPDATE issues SET comments_count = ? WHERE id = ?", statements.length, issueId), ]); await registerRepoRows("comment", ids, repoId); await run("UPDATE integration_import_items SET comments_done = 1 WHERE issue_id = ?", issueId); } /** * Imports one item. The issue lands in the repository's object before its marker in D1, so a crash between them can * import it twice on retry; comments are finished on retry if missing. */ async function importItem( integrationId: string, repoId: string, authorId: string, token: string, githubRepo: string, item: GithubIssue, ): Promise { const seen = await first<{ issue_id: string; comments_done: number }>( "SELECT issue_id, comments_done FROM integration_import_items WHERE integration_id = ? AND source_number = ?", integrationId, item.number, ); if (seen) { if (seen.comments_done === 0) { await applyLabels(repoId, seen.issue_id, item.pull_request ? ["imported", "pull-request"] : ["imported"]); const issue = await repoDb(repoId).first<{ title: string; body: string }>( "SELECT title, body FROM issues WHERE id = ?", seen.issue_id, ); if (issue) await indexIssue({ id: seen.issue_id, repo_id: repoId, ...issue }); await importComments(token, githubRepo, item, repoId, seen.issue_id, authorId); } return false; } const isPull = Boolean(item.pull_request); const number = await allocateIssueNumber(repoId); const id = scopedId(repoId); const created = Date.parse(item.created_at) || Date.now(); const closed = item.closed_at ? Date.parse(item.closed_at) || Date.now() : null; const merged = isPull && item.pull_request?.merged_at ? " (merged)" : ""; const body = clipBody( attribution(isPull ? `Pull request${merged}` : "Issue", item.user?.login ?? null, item.html_url, item.created_at) + (item.body ?? ""), ); await repoDb(repoId).run( `INSERT INTO issues (id, repo_id, number, author_id, title, body, state, is_pull, closed_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)`, id, repoId, number, authorId, item.title.slice(0, 256) || `GitHub #${item.number}`, body, item.state, closed, created, closed ?? created, ); await registerLocator("issue", id, repoId); await batch([ stmt( "INSERT INTO integration_import_items (integration_id, source_number, issue_id) VALUES (?, ?, ?)", integrationId, item.number, id, ), ...(item.state === "open" ? [stmt("UPDATE repos SET open_issues_count = open_issues_count + 1 WHERE id = ?", repoId)] : []), ]); await applyLabels(repoId, id, isPull ? ["imported", "pull-request"] : ["imported"]); await indexIssue({ id, repo_id: repoId, title: item.title, body }); await importComments(token, githubRepo, item, repoId, id, authorId); return true; } /** One page of a one-shot GitHub issue/PR import. Each page re-queues the next, so no invocation runs long. */ export async function runImportPage(message: ImportJobMessage): Promise { const job = await first("SELECT * FROM integration_jobs WHERE id = ?", message.jobId); if (!job || job.status === "done" || job.status === "failed") return; const progress = JSON.parse(job.progress || "{}") as Progress; const row = await loadIntegration(job.integration_id); const config = row ? parseConfig(row) : null; const repo = row && row.scope_type === "repo" ? await loadRepoById(row.scope_id) : null; if (!row || !config || !repo || !row.created_by || !config.githubInstallationId || !config.githubRepo) { await updateJob(job.id, "failed", progress, "The integration is not a configured repository GitHub integration"); return; } if (!isGithubRepoName(config.githubRepo)) { await updateJob(job.id, "failed", progress, "Invalid GitHub repository"); return; } try { await assertMirrorPullAllowed(row, repo); } catch (error) { // A policy refusal ends the import; a GitHub or network failure is rethrown so the queue retries the page. if (!(error instanceof HttpError)) throw error; await updateJob(job.id, "failed", progress, error.message); return; } await updateJob(job.id, "running", progress); if (message.page === 1) { await repoDb(repo.id).batch( IMPORT_LABELS.map((label) => scopedStmt( "INSERT OR IGNORE INTO labels (id, repo_id, name, color, description) VALUES (?, ?, ?, ?, ?)", randomId(), repo.id, label.name, label.color, "Imported from GitHub", ), ), ); } const token = await installationToken(config.githubInstallationId); const items = await githubApi( token, `/repos/${config.githubRepo}/issues?state=all&sort=created&direction=asc&per_page=${PAGE_SIZE}&page=${message.page}`, MAX_PAGE_BYTES, ); let imported = progress.imported ?? 0; for (const item of items) { if (imported >= MAX_IMPORT_ITEMS) break; if (await importItem(row.id, repo.id, row.created_by, token, config.githubRepo, item)) imported++; } const next: Progress = { imported, pages: message.page }; if (items.length < PAGE_SIZE || imported >= MAX_IMPORT_ITEMS) { await updateJob(job.id, "done", { ...next, done: true }); return; } await updateJob(job.id, "running", next); const follow: ImportJobMessage = { kind: "integration.job", jobId: job.id, page: message.page + 1, }; await env.INTEGRATIONS.send(follow); } export async function failImportJob(jobId: string, error: string): Promise { await run( "UPDATE integration_jobs SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status != 'done'", error.slice(0, 500), Date.now(), jobId, ); }