import { recordArtifactsCall } from "./artifacts-health"; import { env } from "cloudflare:workers"; import { first } from "../db"; import { basicAuthHeader, repoTokenSecret } from "./artifacts"; import { concatBytes, packetText, parsePacket, pktLine } from "./pktline"; import { parsePack } from "./pack"; import { limitStream, PackTooLargeError } from "./ref-sync-protocol"; const DELIM_PKT = new TextEncoder().encode("0001"); const FLUSH = new TextEncoder().encode("0000"); const SIDEBAND_DATA = 1; const SIDEBAND_ERROR = 3; const MAX_SNAPSHOT_BYTES = 32 * 1024 * 1024; const MAX_SNAPSHOT_OBJECTS = 200_000; const MAX_TREE_OBJECT_BYTES = 8 * 1024 * 1024; const SNAPSHOT_TIMEOUT_MS = 20_000; const REMOTE_CACHE_SECONDS = 60 * 60; const SHA_BYTES = 20; const decoder = new TextDecoder(); export type TreeListing = Map; export type Snapshot = { trees: TreeListing; blobs: Map }; /** Inflated bytes kept in memory for a snapshot with contents; larger trees fall back to per-blob reads. */ const MAX_SNAPSHOT_INFLATED_BYTES = 32 * 1024 * 1024; const MAX_SNAPSHOT_BLOB_BYTES = 16 * 1024 * 1024; function entryType(mode: string): ArtifactsTreeEntryType { if (mode === "40000") return "tree"; if (mode === "160000") return "gitlink"; if (mode === "120000") return "symlink"; if (mode === "100755") return "exec"; return "blob"; } /** Decodes a raw git tree object into the same shape `readTree` returns. */ export function parseTreeObject(content: Uint8Array): ArtifactsTreeEntry[] { const entries: ArtifactsTreeEntry[] = []; let offset = 0; while (offset < content.byteLength) { const space = content.indexOf(0x20, offset); if (space === -1) throw new Error("Malformed tree object"); const nul = content.indexOf(0, space); if (nul === -1 || nul + 1 + SHA_BYTES > content.byteLength) throw new Error("Malformed tree object"); const mode = decoder.decode(content.subarray(offset, space)); const name = decoder.decode(content.subarray(space + 1, nul)); const hash = [...content.subarray(nul + 1, nul + 1 + SHA_BYTES)] .map((byte) => byte.toString(16).padStart(2, "0")) .join(""); entries.push({ name, mode, hash, type: entryType(mode) }); offset = nul + 1 + SHA_BYTES; } return entries; } export async function remoteFor(artifactsName: string): Promise { const key = `artifacts-remote:${artifactsName}`; const cached = await env.KV.get(key); if (cached) return cached; const row = await first<{ artifacts_remote: string }>( "SELECT artifacts_remote FROM repos WHERE artifacts_name = ?", artifactsName, ); if (!row?.artifacts_remote) return null; await env.KV.put(key, row.artifacts_remote, { expirationTtl: REMOTE_CACHE_SECONDS }); return row.artifacts_remote; } /** Extracts the packfile from a protocol v2 fetch response (section headers plus side-band packets). */ export function packFromResponse(body: Uint8Array): Uint8Array { const chunks: Uint8Array[] = []; let offset = 0; let inPack = false; while (offset < body.byteLength) { const parsed = parsePacket(body, offset); if (!parsed) break; offset = parsed.next; const { packet } = parsed; if (packet.kind !== "data") continue; if (!inPack) { inPack = packetText(packet) === "packfile"; continue; } const band = packet.data[0]; if (band === SIDEBAND_DATA) chunks.push(packet.data.subarray(1)); else if (band === SIDEBAND_ERROR) throw new Error(`Artifacts fetch failed: ${decoder.decode(packet.data.subarray(1))}`); } if (chunks.length === 0) throw new Error("Artifacts fetch returned no packfile"); return concatBytes(chunks); } /** * Fetches every tree beneath `treeSha` in ONE request: a protocol v2 `fetch` that wants the tree itself * with `filter blob:none`, so Artifacts returns all subtrees and no file contents. This replaces one * `readTree` binding call per directory, keeping subrequests flat however large the repository is. * With `blobs`, file contents come too (archives, indexing, deploys), bounded in inflated size. * Returns null when the repository or the response cannot be used, so callers fall back to binding reads. */ export async function fetchTreeSnapshot( artifactsName: string, treeSha: string, options: { blobs?: boolean } = {}, ): Promise { const remote = await remoteFor(artifactsName); if (!remote) return null; const body = concatBytes([ pktLine("command=fetch\n"), pktLine("object-format=sha1\n"), DELIM_PKT, pktLine("no-progress\n"), ...(options.blobs ? [] : [pktLine("filter blob:none\n")]), pktLine(`want ${treeSha}\n`), pktLine("done\n"), FLUSH, ]); // Internal git fetches are billed as pulls; count them next to the binding calls. recordArtifactsCall("git fetch: tree snapshot"); const response = await fetch(`${remote}/git-upload-pack`, { method: "POST", body, headers: { authorization: basicAuthHeader(await repoTokenSecret({ artifacts_name: artifactsName }, "read")), "git-protocol": "version=2", "content-type": "application/x-git-upload-pack-request", accept: "application/x-git-upload-pack-result", "user-agent": "git/gitfor", }, signal: AbortSignal.timeout(SNAPSHOT_TIMEOUT_MS), }); if (!response.ok || !response.body) { await response.body?.cancel(); return null; } if (Number(response.headers.get("content-length") ?? "0") > MAX_SNAPSHOT_BYTES) { await response.body.cancel(); return null; } let bytes: Uint8Array; try { bytes = new Uint8Array(await new Response(limitStream(response.body, MAX_SNAPSHOT_BYTES)).arrayBuffer()); } catch (error) { if (error instanceof PackTooLargeError) return null; throw error; } const parsed = await parsePack(packFromResponse(bytes), { maxObjects: MAX_SNAPSHOT_OBJECTS, maxObjectBytes: options.blobs ? MAX_SNAPSHOT_BLOB_BYTES : MAX_TREE_OBJECT_BYTES, maxTotalBytes: options.blobs ? MAX_SNAPSHOT_INFLATED_BYTES : MAX_SNAPSHOT_BYTES * 4, }); if (parsed.truncated || parsed.unresolved.length > 0) return null; const trees: TreeListing = new Map(); const blobs = new Map(); for (const object of parsed.objects) { if (object.type === "tree") trees.set(object.sha, parseTreeObject(object.content)); else if (object.type === "blob") blobs.set(object.sha, object.content); } return trees.has(treeSha) ? { trees, blobs } : null; }