import { env, waitUntil } from "cloudflare:workers"; import { kvPut } from "./cache"; import { sha256Hex } from "../crypto"; import { withStore } from "../git/store"; import type { GitStore } from "../git/store"; import { branchSha, listRefs } from "../git/refs"; import { loadRepoById } from "../repos"; import { toBrief } from "~/lib/code-intel"; import type { CommitBrief } from "~/lib/code-intel"; import type { CommitSummary } from "~/lib/types"; const PAGE_SIZE = 30; const WALK_BUDGET_PER_REQUEST = 300; const PREFETCH_COMMITS = 100; /** Extra budgets spent after the response is sent, so deep walks finish without the client polling. */ const BACKGROUND_BUDGETS = 10; const MAX_PATH_MATCHES = 1000; const MAX_INLINE_KEY_PATH = 200; /** KV keys are capped at 512 bytes, so long paths are hashed. */ export async function pathKey(path: string): Promise { return path.length <= MAX_INLINE_KEY_PATH ? path : `h:${await sha256Hex(path)}`; } async function pathSha(store: GitStore, commit: CommitSummary, path: string): Promise { const entry = await store.entryAtPath(commit.treeSha, path); return entry?.sha ?? null; } type PathLogState = { matches: CommitBrief[]; cursor: string | null; cursorPathSha: string | null }; export type PathHistoryPage = { commits: CommitBrief[]; hasMore: boolean; capped: boolean }; /** * First-parent history of commits that changed `path`. The walk resumes from a KV-stored cursor so * each request reads at most WALK_BUDGET_PER_REQUEST commits, however deep the requested page. */ export async function pathHistory( store: GitStore, headSha: string, path: string, page: number, ): Promise { const key = `path-log:v2:${store.artifactsName}:${headSha}:${await pathKey(path)}`; const stored = await env.KV.get(key, "json"); const state: PathLogState = stored ?? { matches: [], cursor: headSha, cursorPathSha: null }; const needed = page * PAGE_SIZE + 1; let budget = WALK_BUDGET_PER_REQUEST; let advanced = false; while (state.cursor && state.matches.length < needed && state.matches.length < MAX_PATH_MATCHES && budget-- > 0) { advanced = true; const commit = await store.commit(state.cursor); const own = commit ? (state.cursorPathSha ?? (await pathSha(store, commit, path))) : null; if (!commit || own === null) { state.cursor = null; break; } const parent = commit.parents[0] ? await store.commit(commit.parents[0]) : null; const parentOwn = parent ? (parent.treeSha === commit.treeSha ? own : await pathSha(store, parent, path)) : null; if (own !== parentOwn) state.matches.push(toBrief(commit)); state.cursor = parent && parentOwn !== null ? parent.sha : null; state.cursorPathSha = parentOwn; } if (advanced) await kvPut(key, state); const start = (page - 1) * PAGE_SIZE; const capped = state.matches.length >= MAX_PATH_MATCHES; return { commits: state.matches.slice(start, start + PAGE_SIZE), hasMore: state.matches.length > start + PAGE_SIZE || (state.cursor !== null && !capped), capped, }; } type DirListing = { sha: string; entries: Record }; async function dirListing(store: GitStore, commit: CommitSummary, dir: string): Promise { const entry = await store.entryAtPath(commit.treeSha, dir); if (!entry || entry.type !== "tree") return null; const entries = (await store.tree(entry.sha)) ?? []; return { sha: entry.sha, entries: Object.fromEntries(entries.map((item) => [item.name, item.hash])) }; } type LastCommitState = { result: Record; pending: string[]; cursor: string | null; cursorDir: DirListing | null; /** A first-parent ancestor whose listing is already resolved; the walk stops there and inherits it. */ seed?: { sha: string; result: Record } | null; }; export type LastCommitsPage = { commits: Record; complete: boolean }; /** * Advances the first-parent walk by at most WALK_BUDGET_PER_REQUEST commits, resolving the latest * commit for each directory entry. Progress is kept in KV so repeated calls reach the repository's * first commit however deep the history is. */ async function advanceLastCommits(store: GitStore, path: string, state: LastCommitState): Promise { const pending = new Set(state.pending); if (state.cursor) await store.prefetchHistory(state.cursor, PREFETCH_COMMITS); let current = state.cursor ? await store.commit(state.cursor) : null; let currentDir = state.cursorDir; for (let step = 0; step < WALK_BUDGET_PER_REQUEST && pending.size > 0 && current && currentDir; step++) { if (state.seed && current.sha === state.seed.sha) { for (const name of pending) state.result[name] = state.seed.result[name] ?? null; pending.clear(); break; } const parent: CommitSummary | null = current.parents[0] ? await store.commit(current.parents[0]) : null; const parentDir: DirListing | null = !parent ? null : parent.treeSha === current.treeSha ? currentDir : await dirListing(store, parent, path); if (!parentDir || parentDir.sha !== currentDir.sha) { for (const name of [...pending]) { if (parentDir?.entries[name] === currentDir.entries[name]) continue; state.result[name] = toBrief(current); pending.delete(name); } } current = parent; currentDir = parentDir; } return { result: state.result, pending: [...pending], cursor: pending.size > 0 && current && currentDir ? current.sha : null, cursorDir: pending.size > 0 ? currentDir : null, seed: pending.size > 0 ? state.seed : null, }; } /** Latest first-parent commit touching each entry of a directory, keyed by entry name; resumable across calls. */ const SEED_DEPTH = 8; /** Finds the nearest of the first few first-parent ancestors whose listing was already fully resolved. */ async function findSeed( store: GitStore, head: CommitSummary, keyFor: (sha: string) => string, ): Promise { const chain: string[] = []; let cursor: string | undefined = head.parents[0]; while (cursor && chain.length < SEED_DEPTH) { chain.push(cursor); cursor = (await store.commit(cursor))?.parents[0]; } const states = await Promise.all(chain.map((sha) => env.KV.get(keyFor(sha), "json"))); const index = states.findIndex((candidate) => candidate !== null && candidate.cursor === null); const found = index >= 0 ? states[index] : null; const sha = chain[index]; return found && sha ? { sha, result: found.result } : null; } export async function lastCommitsForDirectory( store: GitStore, headSha: string, path: string, ): Promise { const suffix = await pathKey(path); const keyFor = (sha: string) => `last-commit:v2:${store.artifactsName}:${sha}:${suffix}`; const key = keyFor(headSha); let state = await env.KV.get(key, "json"); if (!state) { const head = await store.commit(headSha); const listing = head ? await dirListing(store, head, path) : null; if (!head || !listing) return { commits: {}, complete: true }; const names = Object.keys(listing.entries); state = { result: Object.fromEntries(names.map((name) => [name, null])), pending: names, cursor: head.sha, cursorDir: listing, seed: await findSeed(store, head, keyFor), }; } if (state.cursor) { state = await advanceLastCommits(store, path, state); await kvPut(key, state); if (state.cursor) waitUntil(continueOnce(store.artifactsName, key, path, state)); } return { commits: state.result, complete: state.cursor === null }; } const walking = new Set(); /** * One background walk per key per isolate. KV can't give mutual exclusion (not atomic, eventually consistent), * so this only removes the common case of one isolate spawning a walk per concurrent request. */ async function continueOnce(artifactsName: string, key: string, path: string, state: LastCommitState) { if (walking.has(key)) return; walking.add(key); try { await continueLastCommits(artifactsName, key, path, state); } finally { walking.delete(key); } } async function continueLastCommits(artifactsName: string, key: string, path: string, start: LastCommitState) { let state = start; try { await withStore(artifactsName, async (store) => { for (let budget = 0; budget < BACKGROUND_BUDGETS && state.cursor; budget++) { state = await advanceLastCommits(store, path, state); await kvPut(key, state); } }); } catch (error) { console.warn("last-commit background walk stopped", error); } } const WARM_ROUNDS = 30; /** Resolves the root listing's last commits for the default branch head so the first tree view is instant. */ export async function warmLastCommits(repoId: string): Promise { const repo = await loadRepoById(repoId); if (!repo) return; const head = branchSha(await listRefs(repo), repo.default_branch); if (!head) return; await withStore(repo.artifacts_name, async (store) => { for (let round = 0; round < WARM_ROUNDS; round++) { const page = await lastCommitsForDirectory(store, head, ""); if (page.complete) return; } }); }