import { env, waitUntil } from "cloudflare:workers"; import { openArtifactsRepo } from "./artifacts"; import type { CommitSummary, TreeEntry } from "~/lib/types"; import { CallLimiter, artifactsCall } from "./artifacts-call"; import { fetchTreeSnapshot } from "./snapshot"; const OBJECT_CACHE_TTL_SECONDS = 60 * 60 * 24 * 30; const SHORT_SHA_LENGTH = 7; const WALK_CONCURRENCY = 16; /** Walks needing more than this many tree reads fetch every tree in one upload-pack request instead. */ const SNAPSHOT_THRESHOLD_READS = 8; const MAX_CACHED_BLOB_BYTES = 64 * 1024; const MAX_WALK_TREE_READS = 2000; export const ZERO_SHA = "0000000000000000000000000000000000000000"; const SHA_PATTERN = /^[0-9a-f]{40}$/; export function isSha(value: string): boolean { return SHA_PATTERN.test(value); } export function shortSha(sha: string): string { return sha.slice(0, SHORT_SHA_LENGTH); } export function toCommitSummary(meta: ArtifactsCommitMetadata): CommitSummary { return { sha: meta.hash, shortSha: shortSha(meta.hash), subject: meta.message.split("\n", 1)[0] ?? "", message: meta.message, authorName: meta.author.name, authorEmail: meta.author.email, committerName: meta.committer.name, authoredAt: meta.authoredAt * 1000, committedAt: meta.committedAt * 1000, parents: meta.parents, treeSha: meta.treeHash, }; } function joinPath(parent: string, name: string): string { return parent ? `${parent}/${name}` : name; } /** Read-side access to a single Artifacts repo with request memoization and KV caching of immutable objects. */ export class GitStore { private readonly commitMemo = new Map>(); private readonly treeMemo = new Map>(); private readonly blobMemo = new Map(); private readonly limiter = new CallLimiter(); private pendingHandle: Promise | null = null; private closed = false; private constructor(readonly artifactsName: string) {} /** * The Artifacts handle is opened on the first read that misses every cache: `get()` is a control-plane * round trip (rate limited per namespace), and many requests are served entirely from KV. */ static open(artifactsName: string): Promise { return Promise.resolve(new GitStore(artifactsName)); } private label(object: string): string { return `${this.artifactsName}@${object.slice(0, 12)}`; } private handle(): Promise { if (this.closed) return Promise.reject(new Error("GitStore is closed")); if (!this.pendingHandle) { const opening = openArtifactsRepo(this.artifactsName); this.pendingHandle = opening; opening.catch(() => { if (this.pendingHandle === opening) this.pendingHandle = null; }); } return this.pendingHandle; } private async withHandle(call: (handle: ArtifactsRepo) => Promise): Promise { const handle = await this.handle(); return call(handle); } close(): void { this.closed = true; const pending = this.pendingHandle; this.pendingHandle = null; pending?.then( (handle) => handle[Symbol.dispose](), () => undefined, ); } private async cached(key: string, load: () => Promise): Promise { const hit = await env.KV.get(key, "json"); if (hit !== null) return hit; const value = await load(); if (value !== null) { waitUntil(env.KV.put(key, JSON.stringify(value), { expirationTtl: OBJECT_CACHE_TTL_SECONDS })); } return value; } commit(sha: string): Promise { const existing = this.commitMemo.get(sha); if (existing) return existing; const promise = this.cached(`commit:${this.artifactsName}:${sha}`, async () => { const meta = await artifactsCall( "read commit", this.label(sha), () => this.withHandle((handle) => handle.readCommit(sha)), { idempotent: true, limiter: this.limiter, }, ); return meta ? toCommitSummary(meta) : null; }); this.commitMemo.set(sha, promise); return promise; } tree(sha: string): Promise { const existing = this.treeMemo.get(sha); if (existing) return existing; const promise = this.cached(`tree:${this.artifactsName}:${sha}`, () => artifactsCall("read tree", this.label(sha), () => this.withHandle((handle) => handle.readTree(sha)), { idempotent: true, limiter: this.limiter, }), ); this.treeMemo.set(sha, promise); return promise; } /** Small blobs are cached in KV by SHA (immutable), like trees and commits. */ async blob(sha: string): Promise | null> { const prefetched = this.blobMemo.get(sha); if (prefetched) return new Uint8Array(prefetched); const key = `blob:${this.artifactsName}:${sha}`; const hit = await env.KV.get(key, "arrayBuffer"); if (hit !== null) return new Uint8Array(hit); const blob = await artifactsCall( "read blob", this.label(sha), () => this.withHandle((handle) => handle.readBlob(sha)), { idempotent: true, limiter: this.limiter, }, ); if (!blob) return null; const bytes = new Uint8Array(await blob.arrayBuffer()); if (bytes.byteLength <= MAX_CACHED_BLOB_BYTES) { waitUntil(env.KV.put(key, bytes, { expirationTtl: OBJECT_CACHE_TTL_SECONDS })); } return bytes; } /** * Loads every tree beneath `treeSha` with one upload-pack request and seeds the request memo, so the * walk below reads from memory. Falls back silently to per-tree binding reads if the fetch fails. */ private async prefetchTrees(treeSha: string, blobs = false): Promise { try { const snapshot = await fetchTreeSnapshot(this.artifactsName, treeSha, { blobs }); if (!snapshot) return false; for (const [sha, entries] of snapshot.trees) { if (!this.treeMemo.has(sha)) this.treeMemo.set(sha, Promise.resolve(entries)); } for (const [sha, bytes] of snapshot.blobs) this.blobMemo.set(sha, bytes); return true; } catch (error) { console.warn("tree snapshot failed; reading trees one by one", this.artifactsName, error); return false; } } async blobStream(sha: string): Promise { return artifactsCall("read blob", this.label(sha), () => this.withHandle((handle) => handle.readBlob(sha)), { idempotent: true, limiter: this.limiter, }); } /** * Fills the commit memo from one `log` call so a first-parent walk costs one Artifacts round trip per * batch instead of one per commit. Entries already memoized or cached are left alone. */ async prefetchHistory(sha: string, limit: number): Promise { if (this.commitMemo.has(sha)) return; const commits = await this.log(sha, limit, 0).catch((): CommitSummary[] => []); for (const commit of commits) { if (!this.commitMemo.has(commit.sha)) this.commitMemo.set(commit.sha, Promise.resolve(commit)); } } async log(ref: string, limit: number, offset: number): Promise { const entries = await artifactsCall( "read history", this.label(ref), () => this.withHandle((handle) => handle.log({ ref, limit, offset })), { idempotent: true, limiter: this.limiter }, ); return entries.map(toCommitSummary); } async listTree(treeSha: string, basePath: string): Promise { const entries = (await this.tree(treeSha)) ?? []; return entries.map((entry) => ({ name: entry.name, path: joinPath(basePath, entry.name), mode: entry.mode, sha: entry.hash, type: entry.type, })); } /** Resolves a slash-separated path under a root tree. An empty path resolves to the root tree itself. */ async entryAtPath(rootTreeSha: string, path: string): Promise { const parts = path.split("/").filter(Boolean); let current: TreeEntry = { name: "", path: "", mode: "40000", sha: rootTreeSha, type: "tree" }; for (const part of parts) { if (current.type !== "tree") return null; const entries = await this.tree(current.sha); const match = entries?.find((entry) => entry.name === part); if (!match) return null; current = { name: match.name, path: joinPath(current.path, match.name), mode: match.mode, sha: match.hash, type: match.type, }; } return current; } /** * Recursively lists every non-tree entry beneath a tree, bounded by `limit`. `withContents` prefetches * file contents in the same single request, for callers that will read most blobs (archives, indexing). * Prefetched blobs live as long as this store (one request). */ async walk( treeSha: string, limit: number, options: { withContents?: boolean } = {}, ): Promise<{ entries: TreeEntry[]; truncated: boolean }> { const withContents = options.withContents === true && (await this.prefetchTrees(treeSha, true)); const out: TreeEntry[] = []; const queue: Array<{ sha: string; path: string }> = [{ sha: treeSha, path: "" }]; let treeReads = 0; let prefetched = withContents; while (queue.length > 0) { if (!prefetched && treeReads >= SNAPSHOT_THRESHOLD_READS) { prefetched = true; await this.prefetchTrees(treeSha); } const level = queue.splice(0, WALK_CONCURRENCY); treeReads += level.length; if (treeReads > MAX_WALK_TREE_READS) return { entries: out, truncated: true }; const listings = await Promise.all(level.map((item) => this.listTree(item.sha, item.path))); for (const listing of listings) { for (const entry of listing) { if (entry.type === "tree") { queue.push({ sha: entry.sha, path: entry.path }); } else { out.push(entry); if (out.length >= limit) return { entries: out, truncated: true }; } } } } return { entries: out, truncated: false }; } } export async function withStore(artifactsName: string, fn: (store: GitStore) => Promise): Promise { const store = await GitStore.open(artifactsName); try { return await fn(store); } finally { store.close(); } }