import { assertAccountActive } from "../account-disable"; import { captureServerEvent } from "../analytics"; import { accountDb, repoDb, type ScopedDb } from "../scoped-db"; import { forgetLocator, locate, registerLocator } from "../scoped-db/locators"; import { scopedId } from "../scoped-db/ids"; import { ownerRepoIds, repoOwnerId, runnerDb } from "./scope"; import { randomToken, sha256Hex } from "../crypto"; import { assertBelowCap, CAPS } from "../limits"; import { forbidden, HttpError } from "../errors"; import { canAdministerAccount } from "../repos"; import { runnerStub } from "./jobs"; import { failStrandedForkJobs } from "./runner-policy"; import type { RepoRow, Viewer } from "~/lib/types"; export const RUNNER_TOKEN_PREFIX = "gitfor_runner"; export const REGISTRATION_TOKEN_PREFIX = "gitfor_reg"; export const REGISTRATION_TTL_MS = 60 * 60 * 1000; /** A rotated-out credential keeps working this long, or until the new one has been used for a while. */ export const ROTATION_GRACE_MS = 60 * 60 * 1000; /** Once the new credential is used, requests already in flight on the old one still get this long. */ const CONFIRMED_GRACE_MS = 5 * 60 * 1000; const TOKEN_HASH = /^[0-9a-f]{64}$/; /** Credentials older than this are asked to rotate on their next request. */ export const ROTATE_AFTER_MS = 24 * 60 * 60 * 1000; export const ONLINE_WINDOW_MS = 2 * 60 * 1000; export const MAX_RUNNER_JOBS = 16; const MAX_LABELS = 20; const MAX_PENDING_REGISTRATIONS = 20; const LABEL = /^[a-z0-9][a-z0-9._-]{0,31}$/; const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const OS_VALUES = new Set(["linux", "macos", "windows", "freebsd"]); const ARCH_VALUES = new Set(["x64", "arm64", "arm", "x86"]); const ISOLATION_VALUES = new Set(["docker", "podman", "none"]); export type RunnerScope = { kind: "repository"; repoId: string } | { kind: "owner"; ownerId: string }; export type RunnerRow = { id: string; repo_id: string | null; owner_id: string | null; name: string; labels: string; os: string; arch: string; version: string; isolation: string; isolation_attested: number; max_jobs: number; token_hash: string; previous_token_hash: string | null; previous_token_expires_at: number | null; token_rotated_at: number | null; token_confirmed: number; last_seen_at: number | null; created_at: number; }; export type RunnerSummary = { id: string; name: string; labels: string[]; os: string; arch: string; version: string; isolation: string; attested: boolean; maxJobs: number; busyJobs: number; lastSeenAt: number | null; online: boolean; scope: "repository" | "owner"; }; /** Normalizes labels and drops anything outside the label grammar; the server, not the daemon, decides labels. */ export function normalizeLabels(labels: readonly string[]): string[] { const out = new Set(); for (const raw of labels) { const label = raw.trim().toLowerCase(); if (LABEL.test(label)) out.add(label); } return [...out].slice(0, MAX_LABELS); } export function scopeColumns(scope: RunnerScope): { repoId: string | null; ownerId: string | null } { return scope.kind === "repository" ? { repoId: scope.repoId, ownerId: null } : { repoId: null, ownerId: scope.ownerId }; } /** * Scope for a runner managed from a repo the viewer administers. Owner-wide runners also serve every other repo * of the owner, so they additionally require administering the owner account. */ export async function repoScope(repo: RepoRow, kind: "repository" | "owner", viewer: Viewer): Promise { if (kind === "repository") return { kind, repoId: repo.id }; if (!(await canAdministerAccount(repo.owner_id, viewer))) throw forbidden("Owner-wide runners require owner admin"); return { kind, ownerId: repo.owner_id }; } /** Runners live in the AccountState of their owner; repository runners in that of the repository's owner. */ export async function runnerScopeAccount(scope: RunnerScope): Promise { if (scope.kind === "owner") return scope.ownerId; const ownerId = await repoOwnerId(scope.repoId); if (!ownerId) throw new HttpError(404, "Repository not found"); return ownerId; } /** In-progress job counts per runner, across the repositories the runners can serve. */ export async function busyRunnerCounts(repoIds: string[], runnerIds: string[]): Promise> { const counts = new Map(); if (runnerIds.length === 0) return counts; const wanted = new Set(runnerIds); const perRepo = await Promise.all( repoIds.map((repoId) => repoDb(repoId).all<{ runner_id: string; count: number }>( `SELECT runner_id, COUNT(*) AS count FROM workflow_jobs WHERE status = 'in_progress' AND runner_id IS NOT NULL GROUP BY runner_id`, ), ), ); for (const row of perRepo.flat()) { if (wanted.has(row.runner_id)) counts.set(row.runner_id, (counts.get(row.runner_id) ?? 0) + row.count); } return counts; } /** Repositories a runner can serve: its own repository, or every repository of its owner. */ export async function runnerRepoIds(runner: Pick): Promise { if (runner.repo_id) return [runner.repo_id]; return runner.owner_id ? ownerRepoIds(runner.owner_id) : []; } async function scopeCount( db: ScopedDb, table: "runners" | "runner_registration_tokens", scope: RunnerScope, ): Promise { const { repoId, ownerId } = scopeColumns(scope); const extra = table === "runner_registration_tokens" ? " AND used_at IS NULL AND expires_at > ?" : ""; const params: (string | number | null)[] = repoId ? [repoId] : [ownerId]; if (extra) params.push(Date.now()); const column = repoId ? "repo_id" : "owner_id"; const row = await db.first<{ count: number }>( `SELECT COUNT(*) AS count FROM ${table} WHERE ${column} = ?${extra}`, ...params, ); return row?.count ?? 0; } /** Isolation the minting admin declares for the machine; the forge never takes the daemon's word for it. */ export type AdminIsolation = "container" | "none"; /** Mints a single-use registration token for a repo or owner; it expires after an hour. */ export async function mintRegistrationToken( scope: RunnerScope, createdBy: string, labels: readonly string[], isolation: AdminIsolation, ): Promise<{ token: string; expiresAt: number }> { const accountId = await runnerScopeAccount(scope); const db = accountDb(accountId); assertBelowCap(await scopeCount(db, "runners", scope), CAPS.runnersPerScope, "runners"); assertBelowCap( await scopeCount(db, "runner_registration_tokens", scope), MAX_PENDING_REGISTRATIONS, "pending runner registration tokens", ); await assertAccountActive(createdBy); const token = randomToken(REGISTRATION_TOKEN_PREFIX); const now = Date.now(); const expiresAt = now + REGISTRATION_TTL_MS; const { repoId, ownerId } = scopeColumns(scope); const hash = await sha256Hex(token); await db.run( `INSERT INTO runner_registration_tokens (token_hash, repo_id, owner_id, created_by, labels, expires_at, created_at, isolation) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, hash, repoId, ownerId, createdBy, JSON.stringify(normalizeLabels(labels)), expiresAt, now, isolation, ); await registerLocator("runner_registration", hash, accountId); return { token, expiresAt }; } export type RegisterInput = { token: string; name: string; labels: string[]; os: string; arch: string; version: string; isolation: string; maxJobs: number; }; function validateRegistration(input: RegisterInput): void { if (!NAME.test(input.name)) throw new HttpError(422, "Runner name must be 1-64 letters, digits, '.', '_' or '-'"); if (!OS_VALUES.has(input.os)) throw new HttpError(422, "Unsupported runner os"); if (!ARCH_VALUES.has(input.arch)) throw new HttpError(422, "Unsupported runner arch"); if (!ISOLATION_VALUES.has(input.isolation)) throw new HttpError(422, "Unsupported isolation mode"); if (!Number.isInteger(input.maxJobs) || input.maxJobs < 1 || input.maxJobs > MAX_RUNNER_JOBS) throw new HttpError(422, `maxJobs must be between 1 and ${MAX_RUNNER_JOBS}`); } /** Consumes a registration token atomically (single use) and creates the runner with a fresh credential. */ export async function registerRunner(input: RegisterInput): Promise<{ runner: RunnerRow; token: string }> { validateRegistration(input); if (!input.token.startsWith(`${REGISTRATION_TOKEN_PREFIX}_`)) throw new HttpError(401, "Invalid registration token"); const hash = await sha256Hex(input.token); const now = Date.now(); const accountId = await locate("runner_registration", hash); if (!accountId) throw new HttpError(401, "Registration token is invalid, expired or already used"); const db = accountDb(accountId); const consumed = await db.run( "UPDATE runner_registration_tokens SET used_at = ? WHERE token_hash = ? AND used_at IS NULL AND expires_at > ?", now, hash, now, ); if (consumed.meta.changes !== 1) throw new HttpError(401, "Registration token is invalid, expired or already used"); const grant = await db.first<{ repo_id: string | null; owner_id: string | null; labels: string; isolation: string }>( "SELECT repo_id, owner_id, labels, isolation FROM runner_registration_tokens WHERE token_hash = ?", hash, ); if (!grant) throw new HttpError(401, "Invalid registration token"); // The admin's declaration bounds the daemon: a container-only token cannot register an unisolated runner, and // only admin-declared container runners count as attested (the gate for fork pull request jobs). if (grant.isolation === "container" && input.isolation === "none") { await db.run("UPDATE runner_registration_tokens SET used_at = NULL WHERE token_hash = ?", hash); throw new HttpError(422, "This registration token requires container isolation (docker or podman)"); } const attested = grant.isolation === "container" ? 1 : 0; const isolation = attested ? input.isolation : "none"; // Labels are fixed by the admin who minted the token; the daemon only reports its os and arch. const labels = normalizeLabels([...JSON.parse(grant.labels), input.os, input.arch]).filter( (label) => label !== "self-hosted", ); const token = randomToken(RUNNER_TOKEN_PREFIX); const id = scopedId(accountId); const scopeColumn = grant.repo_id ? "repo_id" : "owner_id"; const tokenHash = await sha256Hex(token); const inserted = await db.run( `INSERT INTO runners (id, repo_id, owner_id, name, labels, token_hash, created_at, os, arch, version, isolation, max_jobs, token_rotated_at, isolation_attested) SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?7, ?15 WHERE (SELECT COUNT(*) FROM runners WHERE ${scopeColumn} = ?13) < ?14`, id, grant.repo_id, grant.owner_id, input.name, JSON.stringify(labels), tokenHash, now, input.os, input.arch, input.version.slice(0, 64), isolation, input.maxJobs, grant.repo_id ?? grant.owner_id, CAPS.runnersPerScope, attested, ); if (inserted.meta.changes !== 1) { await db.run("UPDATE runner_registration_tokens SET used_at = NULL WHERE token_hash = ?", hash); throw new HttpError(429, `Limit reached: at most ${CAPS.runnersPerScope} runners`); } captureServerEvent( "runner_registered", null, { scope: grant.repo_id ? "repository" : "owner", isolation, attested: attested === 1, // Daemon-reported free text: only plain platform words are kept. os: /^[a-z0-9_]{1,16}$/.test(input.os) ? input.os : "other", arch: /^[a-z0-9_]{1,16}$/.test(input.arch) ? input.arch : "other", }, { ownerAccountId: accountId }, ); const runner = await db.first("SELECT * FROM runners WHERE id = ?", id); if (!runner) throw new HttpError(500, "Failed to register runner"); await registerLocator("runner", id, accountId); await registerLocator("runner_token", tokenHash, accountId); await forgetLocator("runner_registration", hash); return { runner, token }; } /** Looks a credential up by its current hash, or by the previous hash during the rotation grace window. */ export async function runnerForToken(token: string): Promise<{ runner: RunnerRow; previous: boolean } | null> { if (!token.startsWith(`${RUNNER_TOKEN_PREFIX}_`)) return null; const hash = await sha256Hex(token); const accountId = await locate("runner_token", hash); if (!accountId) return null; const db = accountDb(accountId); const current = await db.first("SELECT * FROM runners WHERE token_hash = ?", hash); if (current) { if (current.token_confirmed === 0) { // First use of a rotated credential proves the runner holds it; the previous one only covers requests that // are already in flight from now on. await db.run( `UPDATE runners SET token_confirmed = 1, previous_token_expires_at = MIN(COALESCE(previous_token_expires_at, 0), ?) WHERE id = ? AND token_hash = ?`, Date.now() + CONFIRMED_GRACE_MS, current.id, hash, ); } return { runner: current, previous: false }; } const previous = await db.first( "SELECT * FROM runners WHERE previous_token_hash = ? AND previous_token_expires_at > ?", hash, Date.now(), ); return previous ? { runner: previous, previous: true } : null; } /** * Replaces the credential with one the runner generated and stored before asking, identified by its SHA-256. A * lost response is therefore harmless: the runner already holds the new credential and simply tries it. Only the * current credential may rotate, so a leaked previous credential can never mint a new one. */ export async function rotateRunnerToken(runner: RunnerRow, newTokenSha256: string): Promise { if (!TOKEN_HASH.test(newTokenSha256)) throw new HttpError(422, "token_sha256 must be a hex SHA-256 digest"); const db = await runnerDb(runner.id); if (!db) throw new HttpError(404, "Runner not found"); const rotated = await db.run( `UPDATE runners SET previous_token_hash = token_hash, previous_token_expires_at = ?, token_hash = ?, token_rotated_at = ?, token_confirmed = 0 WHERE id = ? AND token_hash = ?`, Date.now() + ROTATION_GRACE_MS, newTokenSha256, Date.now(), runner.id, runner.token_hash, ); if (rotated.meta.changes !== 1) throw new HttpError(409, "Credential was rotated concurrently"); await registerLocator("runner_token", newTokenSha256, db.scopeId); if (runner.previous_token_hash) await forgetLocator("runner_token", runner.previous_token_hash); } export function needsRotation(runner: RunnerRow): boolean { return Date.now() - (runner.token_rotated_at ?? runner.created_at) > ROTATE_AFTER_MS; } /** Revokes the credential immediately and cancels whatever the runner was executing. */ export async function deleteRunnerRow(runnerId: string): Promise { const db = await runnerDb(runnerId); const row = await db?.first("SELECT * FROM runners WHERE id = ?", runnerId); if (!db || !row) return; const busy = ( await Promise.all( (await runnerRepoIds(row)).map((repoId) => repoDb(repoId).all<{ id: string }>( "SELECT id FROM workflow_jobs WHERE runner_id = ? AND status = 'in_progress' LIMIT ?", runnerId, MAX_RUNNER_JOBS * 4, ), ), ) ).flat(); await db.run("DELETE FROM runners WHERE id = ?", runnerId); await Promise.all( [row.token_hash, row.previous_token_hash] .filter((hash): hash is string => hash !== null) .map((hash) => forgetLocator("runner_token", hash)), ); await forgetLocator("runner", runnerId); await Promise.all(busy.map((job) => runnerStub(job.id).cancel("the runner was removed"))); if (row.repo_id) await failStrandedForkJobs(row.repo_id); } export function summarizeRunner(row: RunnerRow, busyJobs: number): RunnerSummary { return { id: row.id, name: row.name, labels: JSON.parse(row.labels) as string[], os: row.os, arch: row.arch, version: row.version, isolation: row.isolation, attested: row.isolation_attested === 1, maxJobs: row.max_jobs, busyJobs, lastSeenAt: row.last_seen_at, online: row.last_seen_at !== null && Date.now() - row.last_seen_at < ONLINE_WINDOW_MS, scope: row.repo_id ? "repository" : "owner", }; } /** Runners usable by a repo: its own plus its owner's, bounded by the per-scope cap. */ export async function listRepoRunners(repo: RepoRow): Promise { const rows = await accountDb(repo.owner_id).all( "SELECT * FROM runners WHERE repo_id = ? OR owner_id = ? ORDER BY name LIMIT ?", repo.id, repo.owner_id, CAPS.runnersPerScope * 2, ); const repoIds = rows.some((row) => row.repo_id === null) ? await ownerRepoIds(repo.owner_id) : [repo.id]; const counts = await busyRunnerCounts( repoIds, rows.map((row) => row.id), ); return rows.map((row) => summarizeRunner(row, counts.get(row.id) ?? 0)); } /** * Removes (revokes) a runner visible from the repo; owner-scoped runners require owner admin, and a caller whose * credential is limited to this repository (`ownerScopeAllowed` false) can't touch them at all. */ export async function removeRepoRunner( repo: RepoRow, runnerId: string, viewer: Viewer, ownerScopeAllowed = true, ): Promise { const row = await accountDb(repo.owner_id).first<{ repo_id: string | null }>( "SELECT repo_id FROM runners WHERE id = ? AND (repo_id = ? OR owner_id = ?)", runnerId, repo.id, repo.owner_id, ); if (!row) return false; if (row.repo_id === null) { if (!ownerScopeAllowed) throw forbidden("This token cannot manage owner-wide runners"); await repoScope(repo, "owner", viewer); } await deleteRunnerRow(runnerId); return true; }