import { advertisement, artifactsSource, relayPack, RelayScreenRejected } from "../git/relay"; import type { RelaySource } from "../git/relay"; import type { AdvertisedRef, RefCommand } from "../git/ref-sync-protocol"; import { withStore } from "../git/store"; import type { GitStore } from "../git/store"; import type { RawObject } from "../git/objects"; import { pushToArtifacts } from "../git/writer"; import { sourceAuthorization } from "../git/ref-sync"; import { scanPushBody } from "../security/push-protection"; import { badRequest } from "../errors"; import { conflictRef, CONFLICT_PREFIX, decideRef, isMirroredRef, STAGING_PREFIX, stagingRef, ZERO_SHA } from "./plan"; import type { Relation } from "./plan"; import type { ConflictStrategy, MirrorDirection, SyncReport } from "~/lib/mirror"; import { mergeTips, relate } from "./merge"; import { applyLocal, applyRemote } from "./apply"; import type { RepoRow } from "~/lib/types"; const MAX_STAGE_REFS = 200; const MAX_STAGE_BYTES = 1024 * 1024 * 1024; const MAX_COMPARISONS = 100; const MAX_MERGES = 10; export type MirrorSettings = { remoteUrl: string; username: string | null; direction: MirrorDirection; strategy: ConflictStrategy; /** Branch globs (`!` excludes); empty mirrors every branch. */ branches: string[]; syncTags: boolean; propagateDeletes: boolean; /** The admin who configured the mirror; every run acts as them and re-checks their access. */ createdBy: string; }; export type SyncResult = { report: SyncReport; synced: Record; written: Record }; export type Plan = { local: RefCommand[]; remote: RefCommand[]; /** New objects for merge commits, keyed by the ref they land on. */ objects: Map; merged: Set; }; export function mirrorRemote(settings: Pick, token: string | null) { return { url: settings.remoteUrl, authorization: token ? sourceAuthorization(token, settings.username || "git") : null, } satisfies RelaySource; } function emptyReport(): SyncReport { return { pulled: [], pushed: [], merged: [], deletedLocal: [], deletedRemote: [], conflicts: [], refused: [], skipped: 0, }; } const toMap = (refs: AdvertisedRef[]) => new Map(refs.map((ref) => [ref.name, ref.sha])); const isBranchOrTag = (name: string) => name.startsWith("refs/heads/") || name.startsWith("refs/tags/"); /** * Copies the remote's new objects into hidden `refs/gitfor/mirror/*` refs so both tips can be compared and * merged inside the forge. Push protection scans the incoming pack exactly as for a push. Returns the remote * tips that are now available locally. */ async function stageRemote(input: { repo: RepoRow; remote: RelaySource; remoteRefs: Map; staged: Map; present: Set; pusherId: string; report: SyncReport; }): Promise> { const { repo, present } = input; const missing: RefCommand[] = []; for (const [name, sha] of input.remoteRefs) { if (present.has(sha)) continue; missing.push({ name: stagingRef(name), sha, oldSha: input.staged.get(stagingRef(name)) ?? ZERO_SHA }); } const commands = missing.slice(0, MAX_STAGE_REFS); input.report.skipped += missing.length - commands.length; const available = new Set([...input.remoteRefs.values()].filter((sha) => present.has(sha))); if (commands.length === 0) return available; try { const result = await relayPack({ source: input.remote, target: repo, wants: [...new Set(commands.map((command) => command.sha))], haves: [...present], commands, maxBytes: MAX_STAGE_BYTES, screen: async (body) => { const scan = await scanPushBody(repo, input.pusherId, body, false, true, false); const rejected = new Map( scan.rejection ? commands.map((command) => [command.name, "push protection found secrets"]) : [], ); return { body: scan.body, rejected }; }, }); const failed = new Set(result.rejected.map((entry) => entry.split(" ", 1)[0])); for (const command of commands) if (!failed.has(command.name)) available.add(command.sha); } catch (error) { if (error instanceof RelayScreenRejected) { throw badRequest("Push protection found secrets in the remote's new commits; nothing was synced"); } throw error; } return available; } async function planRefs(input: { store: GitStore; settings: MirrorSettings; names: string[]; localRefs: Map; remoteRefs: Map; parked: Map; available: Set; synced: Record; report: SyncReport; }): Promise { const { store, settings, report } = input; const plan: Plan = { local: [], remote: [], objects: new Map(), merged: new Set() }; let comparisons = 0; let merges = 0; const conflict = (name: string, reason: string, paths: string[], local: string | null, remote: string | null) => { const park = name.startsWith("refs/heads/") && remote !== null && settings.direction !== "push"; const parkedAs = park ? conflictRef(name) : null; if (parkedAs && remote && input.parked.get(parkedAs) !== remote) { plan.local.push({ name: parkedAs, sha: remote, oldSha: input.parked.get(parkedAs) ?? ZERO_SHA }); } report.conflicts.push({ ref: name, reason, paths, localSha: local, remoteSha: remote, parkedAs }); }; for (const name of input.names) { const local = input.localRefs.get(name) ?? null; const remote = input.remoteRefs.get(name) ?? null; if (local === remote) continue; if (remote !== null && !input.available.has(remote)) { report.skipped++; continue; } let relation: Relation = "unknown"; if (local && remote && name.startsWith("refs/heads/")) { if (comparisons++ >= MAX_COMPARISONS) { report.skipped++; continue; } relation = await relate(store, local, remote); } const action = decideRef( { name, local, remote, synced: input.synced[name] ?? null, relation }, { direction: settings.direction, strategy: settings.strategy, propagateDeletes: settings.propagateDeletes }, ); if (action.kind === "pull") plan.local.push({ name, sha: remote ?? ZERO_SHA, oldSha: local ?? ZERO_SHA }); else if (action.kind === "push") plan.remote.push({ name, sha: local ?? ZERO_SHA, oldSha: remote ?? ZERO_SHA }); else if (action.kind === "delete-local") plan.local.push({ name, sha: ZERO_SHA, oldSha: local ?? ZERO_SHA }); else if (action.kind === "delete-remote") plan.remote.push({ name, sha: ZERO_SHA, oldSha: remote ?? ZERO_SHA }); else if (action.kind === "conflict") conflict(name, action.reason, [], local, remote); else if (action.kind === "merge" && local && remote) { if (merges++ >= MAX_MERGES) { report.skipped++; continue; } const branch = name.slice("refs/heads/".length); const merge = await mergeTips({ store, branch, local, remote, strategy: settings.strategy }); if (!merge.ok) { conflict(name, merge.reason, merge.paths, local, remote); continue; } plan.local.push({ name, sha: merge.sha, oldSha: local }); plan.remote.push({ name, sha: merge.sha, oldSha: remote }); plan.objects.set(name, merge.objects); plan.merged.add(name); } } const conflicted = new Set(report.conflicts.map((entry) => entry.parkedAs)); for (const [ref, sha] of input.parked) { if (!conflicted.has(ref)) plan.local.push({ name: ref, sha: ZERO_SHA, oldSha: sha }); } return plan; } /** Drops staging refs whose remote ref is gone, so their objects can eventually be collected. */ async function pruneStaging(repo: RepoRow, staged: Map, remoteRefs: Map) { const live = new Set([...remoteRefs.keys()].map(stagingRef)); const stale = [...staged].filter(([ref]) => !live.has(ref)); await pushToArtifacts( repo, stale.map(([ref, sha]) => ({ ref, oldSha: sha, newSha: ZERO_SHA })), [], ); } /** * One bidirectional sync pass. Fetches both ref lists, stages the remote's new objects, plans every ref * (fast-forward, merge, park a conflict, propagate a deletion), applies forge-side updates through the same * gates as a push, then pushes the forge side to the remote. Returns what changed plus the refs both sides * now agree on, which the next pass uses to tell a deletion from a new ref. */ export async function runMirrorSync( repo: RepoRow, settings: MirrorSettings, token: string | null, previous: Record, onWrite?: (written: Map) => Promise, ): Promise { const report = emptyReport(); const remote = mirrorRemote(settings, token); const self = await artifactsSource(repo); const [remoteList, localAll] = await Promise.all([advertisement(remote), advertisement(self, true)]); const scoped = (name: string) => isMirroredRef(name, settings.branches, settings.syncTags); const remoteRefs = toMap(remoteList.filter((ref) => scoped(ref.name))); const localRefs = toMap(localAll.filter((ref) => isBranchOrTag(ref.name) && scoped(ref.name))); const staged = toMap(localAll.filter((ref) => ref.name.startsWith(STAGING_PREFIX))); const parked = toMap(localAll.filter((ref) => ref.name.startsWith(CONFLICT_PREFIX))); const present = new Set(localAll.map((ref) => ref.sha)); const available = await stageRemote({ repo, remote, remoteRefs, staged, present, pusherId: settings.createdBy, report, }); const synced = Object.fromEntries(Object.entries(previous).filter(([name]) => scoped(name))); const names = [...new Set([...remoteRefs.keys(), ...localRefs.keys(), ...Object.keys(synced)])].sort(); const plan = await withStore(repo.artifacts_name, (store) => planRefs({ store, settings, names, localRefs, remoteRefs, parked, available, synced, report }), ); const local = await applyLocal({ repo, plan, pusherId: settings.createdBy, report, onWrite }); const pushed = await applyRemote({ self, remote, plan, haves: [...available], report }); await pruneStaging(repo, staged, remoteRefs).catch((error: unknown) => console.warn("mirror staging prune", error)); for (const name of names) { const finalLocal = local.has(name) ? local.get(name) : localRefs.get(name); const finalRemote = pushed.has(name) ? pushed.get(name) : remoteRefs.get(name); const lsha = finalLocal && finalLocal !== ZERO_SHA ? finalLocal : null; const rsha = finalRemote && finalRemote !== ZERO_SHA ? finalRemote : null; if (lsha && lsha === rsha) synced[name] = lsha; else if (!lsha && !rsha) delete synced[name]; } return { report, synced, written: Object.fromEntries(local) }; }