import { env } from "cloudflare:workers"; import { all } from "./db"; /** * One-shot data backfills that walk every repository once, a page per cron run, so new indexes can cover rows * written before they existed. Progress lives in KV (`backfill:`): the last repository id visited, or * DONE. Readers that need the index consult `backfillDone` and keep their old (slower) path until it is. */ const DONE = "done"; const PAGE = 50; const finished = new Set(); function key(name: string): string { return `backfill:${name}`; } /** False until the backfill has finished, and whenever its state can't be read (callers keep their safe path). */ export async function backfillDone(name: string): Promise { if (finished.has(name)) return true; const state = await Promise.resolve() .then(() => env.KV.get(key(name))) .catch(() => null); if (state === DONE) finished.add(name); return state === DONE; } async function walk(name: string, table: "repos" | "accounts", visit: (id: string) => Promise): Promise { if (await backfillDone(name)) return true; const cursor = (await env.KV.get(key(name))) ?? ""; const rows = await all<{ id: string }>(`SELECT id FROM ${table} WHERE id > ? ORDER BY id LIMIT ?`, cursor, PAGE); for (const row of rows) await visit(row.id); const done = rows.length < PAGE; await env.KV.put(key(name), done ? DONE : (rows.at(-1)?.id ?? cursor)); if (done) finished.add(name); return done; } /** Visits the next page of repositories in id order; call from the cron until it reports done. */ export function backfillRepos(name: string, visit: (repoId: string) => Promise): Promise { return walk(name, "repos", visit); } /** Visits the next page of accounts (users and organizations) in id order. */ export function backfillAccounts(name: string, visit: (accountId: string) => Promise): Promise { return walk(name, "accounts", visit); }