import { waitUntil } from "cloudflare:workers"; import { run } from "../db"; import { BACKFILL_START } from "../repos"; import { advertisement, artifactsSource, assertPublicHttpsUrl, relayPack } from "./relay"; import { pushToArtifacts } from "./writer"; import { assertResolvesPublic } from "../net/resolve-guard"; import { badRequest } from "../errors"; import type { RepoRow } from "~/lib/types"; const MAX_SYNC_REFS = 1000; const MAX_SYNC_BYTES = 1024 * 1024 * 1024; const ZERO_SHA = "0".repeat(40); export type RefSyncResult = { created: number; rejected: string[]; /** Refs still missing after this pass; call again to continue. */ skipped: number; headBranch: string | null; }; export type RefSyncOptions = { /** Copy just the primary branch (`branch`, else the source HEAD) so the repository opens quickly. */ only?: "head"; branch?: string | null; }; /** Basic auth for a source remote: GitHub and GitLab accept any username with a token as the password. */ export function sourceAuthorization(token: string, username = "git"): string { return `Basic ${btoa(`${username}:${token}`)}`; } /** * Copies branches and tags from a git remote by relaying a packfile straight from the source's * upload-pack into Artifacts' receive-pack, so nothing is buffered in the Worker. Each call copies at * most MAX_SYNC_REFS refs and reports how many remain; existing refs are never overwritten. */ export async function syncRefsFromSource( repo: RepoRow, sourceUrl: string, authorization: string | null = null, options: RefSyncOptions = {}, ): Promise { assertPublicHttpsUrl(sourceUrl); await assertResolvesPublic(sourceUrl); const remote = { url: sourceUrl, authorization }; const [sourceAll, target] = await Promise.all([ advertisement(remote, true), artifactsSource(repo).then((self) => advertisement(self)), ]); const head = sourceAll.find((ref) => ref.name === "HEAD")?.sha ?? null; const source = sourceAll.filter((ref) => ref.name.startsWith("refs/heads/") || ref.name.startsWith("refs/tags/")); const preferred = options.branch ? source.find((ref) => ref.name === `refs/heads/${options.branch}`) : null; const headRef = preferred ?? source.find((ref) => ref.name.startsWith("refs/heads/") && ref.sha === head) ?? source.find((ref) => ref.name.startsWith("refs/heads/")) ?? null; const headBranch = headRef?.name.slice("refs/heads/".length) ?? null; if (options.only === "head" && !headRef) throw badRequest("The source has no branches to import"); const existingNames = new Set(target.map((ref) => ref.name)); const wanted = options.only === "head" && headRef ? [headRef] : source; const allMissing = wanted.filter((ref) => !existingNames.has(ref.name)); const missing = allMissing.slice(0, MAX_SYNC_REFS); const result: RefSyncResult = { created: 0, rejected: [], skipped: allMissing.length - missing.length, headBranch }; if (missing.length === 0) return result; const present = new Set(target.map((ref) => ref.sha)); const wants = [...new Set(missing.map((ref) => ref.sha))].filter((sha) => !present.has(sha)); if (wants.length === 0) { await pushToArtifacts( repo, missing.map((ref) => ({ ref: ref.name, oldSha: ZERO_SHA, newSha: ref.sha })), [], ); return { ...result, created: missing.length }; } const report = await relayPack({ source: remote, target: repo, wants, haves: [...present], commands: missing, maxBytes: MAX_SYNC_BYTES, }); return { ...result, created: report.accepted, rejected: report.rejected }; } const RELAY_EVENT_SETTLE_MS = 15_000; /** How long a re-sync's relay window is trusted even if its decrement never runs. */ const RESYNC_WINDOW_MS = 60 * 60 * 1000; /** * Re-syncs an imported repository's refs from its source. Relayed refs raise anonymous Artifacts push events; * `backfilling` (a count of relays in flight, so overlapping syncs can't clear each other's window) tells * PushWorkflow they are an import, not a push, so they run no CI with the repository's secrets, fire no webhooks * or deploys and don't show up as a deleted user in the activity feed. Pushes by people always carry a pusher * and are never taken for relays. */ export async function resyncImportedRefs(repo: RepoRow, sourceUrl: string): Promise { const now = Date.now(); await run(`UPDATE repos SET ${BACKFILL_START} WHERE id = ?3`, now, now + RESYNC_WINDOW_MS, repo.id); try { return await syncRefsFromSource(repo, sourceUrl); } finally { waitUntil( new Promise((resolve) => setTimeout(resolve, RELAY_EVENT_SETTLE_MS)).then(() => run("UPDATE repos SET backfilling = MAX(backfilling - 1, 0) WHERE id = ?", repo.id), ), ); } }