import { env } from "cloudflare:workers"; import { repoDb } from "../scoped-db"; import { jobDb } from "./scope"; import { decryptRepoSecret } from "../secrets"; import { loadRepoById } from "../repos"; import { setCommitStatus } from "../statuses"; import { runUrl, statusContext } from "./scheduler"; import { environmentIdFor, loadSecretRows } from "./config-values"; import { markDeploymentStarted, settleDeployment } from "./deployments"; import { cloudRunnerSize } from "./runner-sizes"; import { withJobDefaults } from "./compile"; import type { CompiledJob } from "./compile"; import { jobSeconds, meterCompletedJob, runnerMeter } from "../metering/ci"; import { captureServerEvent } from "../analytics"; import { releaseJobHold } from "./job-hold"; import { mintHeldRepoToken, releaseRepoToken } from "../git/artifacts"; import type { Conclusion, JobStatus, RepoRow, RunStatus } from "~/lib/types"; export type WorkflowRunRow = { id: string; repo_id: string; source_repo_id: string; run_number: number; workflow_path: string; workflow_name: string; event: string; ref: string; sha: string; title: string; actor_id: string | null; pr_number: number | null; approval_required: number; workflow_source: string | null; context: string | null; trigger_depth: number; rerun_of: string | null; untrusted: number; status: RunStatus; conclusion: Conclusion | null; created_at: number; started_at: number | null; completed_at: number | null; }; export type WorkflowJobRow = { id: string; run_id: string; repo_id: string; job_key: string; name: string; runs_on: string; needs: string; spec: string; status: JobStatus; conclusion: Conclusion | null; runner_id: string | null; runner_name: string | null; outputs: string | null; environment: string | null; gate: string | null; created_at: number; started_at: number | null; completed_at: number | null; }; export type JobExecution = { job: WorkflowJobRow; run: WorkflowRunRow; repo: RepoRow; sourceRepo: RepoRow; spec: CompiledJob; }; export function isCloudJob(runsOn: string[]): boolean { return cloudRunnerSize(runsOn) !== null; } /** Names that resolve to the job's own short-lived token rather than a stored secret. */ const JOB_TOKEN_SECRETS = new Set(["GITHUB_TOKEN", "GITFOR_TOKEN"]); /** Fork pull request runs and merge queue groups containing fork code get no secrets and a read-only token. */ export function isUntrustedRun(execution: Pick): boolean { return execution.sourceRepo.id !== execution.repo.id || execution.run.untrusted === 1; } export async function loadJobExecution(jobId: string): Promise { const db = await jobDb(jobId); if (!db) return null; const job = await db.first("SELECT * FROM workflow_jobs WHERE id = ?", jobId); if (!job) return null; const runRow = await db.first("SELECT * FROM workflow_runs WHERE id = ?", job.run_id); if (!runRow) return null; const [repo, sourceRepo] = await Promise.all([loadRepoById(runRow.repo_id), loadRepoById(runRow.source_repo_id)]); if (!repo || !sourceRepo) return null; return { job, run: runRow, repo, sourceRepo, spec: withJobDefaults(JSON.parse(job.spec)) }; } /** * Decrypts the named secrets (environment, repository and owner scopes). Fork pull request runs never receive * stored secrets; `GITHUB_TOKEN`/`GITFOR_TOKEN` resolve to the job token, which is read-only for forks. */ export async function resolveSecrets( execution: JobExecution, names: string[], jobToken: string | null = null, ): Promise> { const values: Record = Object.fromEntries(names.map((name) => [name, ""])); for (const name of names) if (JOB_TOKEN_SECRETS.has(name) && jobToken) values[name] = jobToken; const wanted = new Set(names.filter((name) => !JOB_TOKEN_SECRETS.has(name))); if (wanted.size === 0 || isUntrustedRun(execution)) return values; const environmentId = await environmentIdFor(execution.repo.id, execution.spec.environment?.name); const rows = await loadSecretRows(execution.repo, environmentId); for (const row of rows.filter((candidate) => wanted.has(candidate.name))) { values[row.name] = await decryptRepoSecret(row.ciphertext, row.iv); } return values; } export async function markJobStarted( execution: JobExecution, runnerId: string | null, runnerName: string, ): Promise { const now = Date.now(); // A job cancelled between dispatch/claim and start stays cancelled instead of being revived as in_progress. const started = await repoDb(execution.job.repo_id).run( `UPDATE workflow_jobs SET status = 'in_progress', started_at = COALESCE(started_at, ?), runner_id = ?, runner_name = ? WHERE id = ? AND status != 'completed'`, now, runnerId, runnerName, execution.job.id, ); if (started.meta.changes !== 1) return false; await markDeploymentStarted(execution.job.id); await announceJob(execution.job.id, execution.job.runs_on, "started"); captureServerEvent("ci_run_started", execution.run.actor_id, ciProperties(execution), { ownerAccountId: execution.repo.owner_id, }); await setCommitStatus({ repoId: execution.repo.id, sha: execution.run.sha, context: statusContext(execution.run.workflow_name, execution.job.name), state: "pending", description: "In progress", targetUrl: runUrl(execution.repo, execution.run.run_number), source: "ci", }); return true; } export async function updateStep( jobId: string, index: number, status: "in_progress" | "completed", conclusion: Conclusion | null, ): Promise { const db = await jobDb(jobId); if (!db) return; const now = Date.now(); if (status === "in_progress") { await db.run( "UPDATE job_steps SET status = 'in_progress', started_at = ? WHERE job_id = ? AND idx = ?", now, jobId, index, ); return; } await db.run( `UPDATE job_steps SET status = 'completed', conclusion = ?, completed_at = ?, started_at = COALESCE(started_at, ?) WHERE job_id = ? AND idx = ?`, conclusion, now, now, jobId, index, ); } const STATUS_FOR_CONCLUSION: Record = { success: "success", skipped: "success", failure: "failure", cancelled: "error", }; /** Enums only: runner labels can be custom self-hosted names, so only the billed size class is sent. */ function ciProperties(execution: JobExecution): Record { return { runner_size: runnerMeter(execution.spec.runsOn).replace(/^compute\.runner_?/, "") || "standard", event: execution.run.event, private_repo: execution.repo.visibility !== "public", fork: execution.sourceRepo.id !== execution.repo.id, }; } async function meterJob(execution: JobExecution, completedAt: number): Promise { try { await meterCompletedJob({ jobId: execution.job.id, runId: execution.run.id, accountId: execution.repo.owner_id, repoId: execution.repo.id, sourceRepoId: execution.sourceRepo.id, runsOn: execution.spec.runsOn, startedAt: execution.job.started_at, completedAt, }); } catch (error) { console.warn("CI metering failed", execution.job.id, error); } } function ciJobTokenOwner(jobId: string): string { return `ci-job:${jobId}`; } /** The job's Artifacts clone token, held until `completeJob` revokes it. */ export async function mintJobCloneToken(jobId: string, artifactsName: string, ttlSeconds: number): Promise { return (await mintHeldRepoToken(ciJobTokenOwner(jobId), artifactsName, "read", ttlSeconds)).secret; } /** Best effort: viewers also poll, so a failed announcement only delays the page by a few seconds. */ async function announceJob(jobId: string, runsOn: string, event: "started" | "finished"): Promise { await runnerStub(jobId, runsOn) .announce(event) .catch((error: unknown) => console.warn("job announcement failed", jobId, error)); } /** Records a terminal job state, resolves its commit status and wakes the orchestrating Workflow. */ export async function completeJob(jobId: string, requested: Conclusion): Promise { const execution = await loadJobExecution(jobId); if (!execution) return; const db = repoDb(execution.job.repo_id); const now = Date.now(); const settled = await db.run( `UPDATE workflow_jobs SET status = 'completed', conclusion = ?, completed_at = ? WHERE id = ? AND status != 'completed'`, requested, now, jobId, ); const firstSettle = settled.meta.changes === 1; // Retries (finalizing alarm, cancelActiveJob, workflow finish) re-run the idempotent cleanup below with the // conclusion that won, so a partial first attempt can't leave tokens, the hold or the run open (CI-A6) and a // late completion can't repaint the job. const conclusion = firstSettle ? requested : (execution.job.conclusion ?? (await storedConclusion(execution.job.repo_id, jobId))); if (!conclusion) return; const completedAt = firstSettle ? now : (execution.job.completed_at ?? now); await db.run( `UPDATE job_steps SET status = 'completed', conclusion = 'skipped', completed_at = ? WHERE job_id = ? AND status = 'pending'`, completedAt, jobId, ); await db.run( `UPDATE job_steps SET status = 'completed', conclusion = ?, completed_at = ? WHERE job_id = ? AND status = 'in_progress'`, conclusion, completedAt, jobId, ); if (firstSettle) { await meterJob(execution, now); await announceJob(jobId, execution.job.runs_on, "finished"); const minutes = Math.round((jobSeconds(execution.job.started_at, now) / 60) * 100) / 100; captureServerEvent( "ci_run_completed", execution.run.actor_id, { ...ciProperties(execution), status: conclusion, minutes }, { ownerAccountId: execution.repo.owner_id }, ); } await releaseJobHold(jobId, execution.repo.owner_id).catch((error: unknown) => console.warn("CI hold release failed", jobId, error), ); await db.run("DELETE FROM job_tokens WHERE job_id = ?", jobId); await releaseRepoToken(ciJobTokenOwner(jobId)); await settleDeployment(jobId, conclusion); // An attempt that was since re-run (late completion, alarm retry) must not paint its stale state over the newer attempt's status. if (!(await supersededAttempt(execution))) await setCommitStatus({ repoId: execution.repo.id, sha: execution.run.sha, context: statusContext(execution.run.workflow_name, execution.job.name), state: STATUS_FOR_CONCLUSION[conclusion], description: conclusion === "success" ? "Successful" : conclusion === "skipped" ? "Skipped" : `Job ${conclusion}`, targetUrl: runUrl(execution.repo, execution.run.run_number), source: "ci", workflowPath: execution.run.workflow_path, workflowSource: execution.run.workflow_source, }); try { const instance = await env.CI_WORKFLOW.get(execution.run.id); await instance.sendEvent({ type: "job-done", payload: { jobId } }); } catch (error) { console.error("Failed to signal workflow run", execution.run.id, error); } } /** * True when a later run of the same workflow on the same commit has a job for the same status context that already * started (so it owns the status now). A later run that is still pending, or was cancelled before starting, doesn't * silence this one, so concurrent runs on one commit can't leave the check pending. Reruns always create a new run * (failed-only reruns copy successful jobs in with their original start time). */ async function supersededAttempt(execution: JobExecution): Promise { const newer = await repoDb(execution.run.repo_id).first<{ one: number }>( `SELECT 1 AS one FROM workflow_runs r JOIN workflow_jobs j ON j.run_id = r.id WHERE r.repo_id = ? AND r.sha = ? AND r.workflow_name = ? AND j.name = ? AND r.run_number > ? AND j.started_at IS NOT NULL LIMIT 1`, execution.run.repo_id, execution.run.sha, execution.run.workflow_name, execution.job.name, execution.run.run_number, ); return newer !== null; } async function storedConclusion(repoId: string, jobId: string): Promise { const row = await repoDb(repoId).first<{ conclusion: Conclusion | null }>( "SELECT conclusion FROM workflow_jobs WHERE id = ?", jobId, ); return row?.conclusion ?? null; } /** * Stops a running job through its runner, then settles the row as cancelled even when the runner had nothing to * stop (claimed but not attached yet, or mid-finalize), so billing and the hold settle now and the job can't * finish as success afterwards. */ export async function cancelActiveJob(job: Pick, reason: string): Promise { await runnerStub(job.id, job.runs_on) .cancel(reason) .catch((error: unknown) => console.error("Runner cancel failed", job.id, error)); await completeJob(job.id, "cancelled"); } /** The Durable Object that owns a job's logs (and, for cloud jobs, its Container of the requested size). */ export function runnerStub(jobId: string, runsOn: string[] | string = []) { const labels: string[] = typeof runsOn === "string" ? JSON.parse(runsOn) : runsOn; const namespace = env[cloudRunnerSize(labels)?.binding ?? "RUNNER"]; return namespace.get(namespace.idFromName(`job:${jobId}`)); }