import { repoDb, stmt } from "../scoped-db"; import { accountsById } from "../people"; import { randomId } from "../crypto"; import { conflict, forbidden, notFoundError } from "../errors"; import { publicUrl } from "../events/emit"; import { concatBytes, parsePacket } from "../git/pktline"; import { applyDelta, parsePack } from "../git/pack"; import type { UnresolvedDelta } from "../git/pack"; import { shortSha, withStore } from "../git/store"; import { detectorName, redactFindings, scannableText, scanText } from "./detectors"; import type { RedactedFinding } from "./detectors"; import { allowedFingerprints } from "./secret-alerts"; import { loadSecuritySettings, nextSecurityNumber } from "./settings"; import { notifyPushProtectionBypass } from "../notify/security"; import type { RepoRow } from "~/lib/types"; const MAX_BUFFERED_PUSH_BYTES = 16 * 1024 * 1024; const MAX_INFLATED_PUSH_BYTES = 48 * 1024 * 1024; const MAX_PACK_OBJECT_BYTES = 8 * 1024 * 1024; const MAX_PACK_OBJECTS = 20_000; const MAX_SCANNED_TOTAL_BYTES = 16 * 1024 * 1024; const MAX_BLOCKS_PER_PUSH = 20; const MAX_THIN_BASES = 500; const PACK_SIGNATURE = "PACK"; export type Buffered = { bytes: Uint8Array; complete: boolean; body: ReadableStream }; /** Reads up to `cap` bytes; when the stream is longer it re-joins the prefix with the unread tail. */ export async function bufferUpTo(stream: ReadableStream, cap: number): Promise { const reader = stream.getReader(); const chunks: Uint8Array[] = []; let total = 0; while (total <= cap) { const { value, done } = await reader.read(); if (done) { const bytes = concatBytes(chunks); return { bytes, complete: true, body: new Blob([bytes]).stream() }; } chunks.push(value); total += value.byteLength; } const prefix = [...chunks]; const body = new ReadableStream({ async pull(controller) { const next = prefix.shift(); if (next) return controller.enqueue(next); const { value, done } = await reader.read(); if (done) controller.close(); else controller.enqueue(value); }, cancel: (reason) => reader.cancel(reason), }); return { bytes: concatBytes(chunks), complete: false, body }; } async function gunzip(bytes: Uint8Array): Promise> { const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip")); const inflated = await bufferUpTo(stream, MAX_INFLATED_PUSH_BYTES); if (!inflated.complete) throw new ScanIncompleteError("the push is too large to scan"); return inflated.bytes; } export class ScanIncompleteError extends Error {} /** Skips the command list (and push-options section) to find where the packfile begins. */ export function packOffset(body: Uint8Array): number | null { let offset = 0; for (let sections = 0; sections < 2; sections++) { while (true) { const parsed = parsePacket(body, offset); if (!parsed) return null; offset = parsed.next; if (parsed.packet.kind === "flush") break; } if (new TextDecoder().decode(body.subarray(offset, offset + 4)) === PACK_SIGNATURE) return offset; } return null; } export type ThinBaseLoader = { blob: (sha: string) => Promise; isNonBlob: (sha: string) => Promise; }; /** * Scans thin-pack deltas against their stored bases. A blob delta whose base cannot be loaded or applied * could hide a secret split across base and delta, so it fails the scan (GIT-P5); deltas against trees * or commits carry no file content, so their inserted literals are enough. */ export async function scanThinDeltas( deltas: UnresolvedDelta[], loader: ThinBaseLoader, scan: (bytes: Uint8Array) => Promise, ): Promise { for (const delta of deltas) { const base = delta.baseSha ? await loader.blob(delta.baseSha) : null; const full = base ? applyDelta(base, delta.delta, MAX_PACK_OBJECT_BYTES) : null; if (full) { await scan(full); continue; } if (base || !delta.baseSha || !(await loader.isNonBlob(delta.baseSha))) { throw new ScanIncompleteError("a delta base in this push could not be loaded"); } for (const insert of delta.inserted) await scan(insert); } } type PushFinding = RedactedFinding & { blobSha: string | null }; async function scanPack(repo: RepoRow, pack: Uint8Array): Promise { const parsed = await parsePack(pack, { maxObjects: MAX_PACK_OBJECTS, maxObjectBytes: MAX_PACK_OBJECT_BYTES, maxTotalBytes: MAX_INFLATED_PUSH_BYTES, }); if (parsed.truncated) throw new ScanIncompleteError("the push has too many or too large objects to scan"); const findings: PushFinding[] = []; let scanned = 0; const scan = async (bytes: Uint8Array, blobSha: string | null) => { const text = scannableText(bytes); if (!text) return; scanned += bytes.byteLength; if (scanned > MAX_SCANNED_TOTAL_BYTES) throw new ScanIncompleteError("the push has too much text to scan"); for (const finding of await redactFindings(scanText(text))) findings.push({ ...finding, blobSha }); }; for (const object of parsed.objects) if (object.type === "blob") await scan(object.content, object.sha); if (parsed.unresolved.length > MAX_THIN_BASES) { throw new ScanIncompleteError("the push has too many deltas against existing objects to scan"); } if (parsed.unresolved.length > 0) { await withStore(repo.artifacts_name, (store) => scanThinDeltas( parsed.unresolved, { blob: (sha) => store.blob(sha).catch(() => null), isNonBlob: async (sha) => (await store.tree(sha).catch(() => null)) !== null || (await store.commit(sha).catch(() => null)) !== null, }, (bytes) => scan(bytes, null), ), ); } return findings; } function blockMessage(repo: RepoRow, blocks: Array<{ id: string; finding: PushFinding }>): string { const lines = ["", "gitfor: PUSH REJECTED — push protection found secrets in this push.", ""]; for (const { id, finding } of blocks) { const where = finding.blobSha ? ` in blob ${shortSha(finding.blobSha)}` : ""; lines.push(` - ${detectorName(finding.detector)} (${finding.preview})${where}, line ${finding.line}`); lines.push( ` review or bypass: ${publicUrl(`/${repo.owner_name}/${repo.name}/security/secret-scanning/unblock/${id}`)}`, ); } lines.push("", "Remove the secret from your commits (e.g. amend or rebase), or bypass it if it is safe.", ""); return lines.join("\n"); } export type PushScan = { body: ReadableStream; rejection: string | null }; /** * Push protection: buffers the receive-pack body (bounded), scans pack blobs and thin-pack deltas * (resolved against the stored base blob) for secrets, and returns a rejection message when unallowed * secrets are found. Fails closed: a push that cannot be scanned completely is rejected, unless * `allowUnscanned` (repository admins), in which case it lands and the post-push scan raises alerts. */ export async function scanPushBody( repo: RepoRow, pusherId: string | null, stream: ReadableStream, gzip: boolean, expectsPack = true, allowUnscanned = false, ): Promise { const settings = await loadSecuritySettings(repo.id); if (!settings.pushProtection || !settings.secretScanning) return { body: stream, rejection: null }; const buffered = await bufferUpTo(stream, MAX_BUFFERED_PUSH_BYTES); const refuse = async (reason: string): Promise => { if (allowUnscanned) { console.warn("push protection skipped for an admin push; relying on the post-push scan", { repo: repo.id, reason, }); return { body: buffered.body, rejection: null }; } await buffered.body.cancel().catch(() => undefined); return { body: buffered.body, rejection: `gitfor: PUSH REJECTED — push protection could not scan this push: ${reason}. Push in smaller pieces, or ask a repository admin to push it (admin pushes fall back to the post-push scan).`, }; }; if (!buffered.complete) return refuse("the push is larger than 16 MB"); let findings: PushFinding[] = []; try { const decoded = gzip ? await gunzip(buffered.bytes) : buffered.bytes; const offset = packOffset(decoded); if (offset !== null) findings = await scanPack(repo, decoded.subarray(offset)); else if (expectsPack) return refuse("no packfile was found"); } catch (error) { console.warn("push protection scan failed", error); return refuse(error instanceof ScanIncompleteError ? error.message : "the packfile could not be read"); } if (findings.length === 0) return { body: buffered.body, rejection: null }; const allowed = await allowedFingerprints( repo.id, findings.map((finding) => finding.fingerprint), pusherId, ); const blocking = [...new Map(findings.map((finding) => [finding.fingerprint, finding])).values()] .filter((finding) => !allowed.has(finding.fingerprint)) .slice(0, MAX_BLOCKS_PER_PUSH); if (blocking.length === 0) return { body: buffered.body, rejection: null }; const now = Date.now(); const blocks = blocking.map((finding) => ({ id: randomId(), finding })); await repoDb(repo.id).batch( blocks.map(({ id, finding }) => stmt( `INSERT INTO secret_push_blocks (id, repo_id, user_id, detector, fingerprint, preview, blob_sha, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, id, repo.id, pusherId, finding.detector, finding.fingerprint, finding.preview, finding.blobSha, now, ), ), ); await buffered.body.cancel().catch(() => undefined); return { body: buffered.body, rejection: blockMessage(repo, blocks) }; } export type PushBlock = { id: string; detector: string; detectorName: string; preview: string; blobSha: string | null; pusher: string | null; bypassedAt: number | null; createdAt: number; }; type BlockRow = { id: string; detector: string; preview: string; blob_sha: string | null; pusher: string | null; fingerprint: string; bypassed_at: number | null; created_at: number; }; export async function loadPushBlock(repoId: string, id: string): Promise { const row = await repoDb(repoId).first & { user_id: string | null }>( `SELECT id, detector, preview, blob_sha, fingerprint, bypassed_at, created_at, user_id FROM secret_push_blocks WHERE repo_id = ? AND id = ?`, repoId, id, ); if (!row) throw notFoundError("Blocked push not found"); const pusher = row.user_id ? ((await accountsById([row.user_id])).get(row.user_id)?.name ?? null) : null; return { id: row.id, detector: row.detector, detectorName: detectorName(row.detector), preview: row.preview, blobSha: row.blob_sha, pusher, bypassedAt: row.bypassed_at, createdAt: row.created_at, }; } export type BypassReason = "false_positive" | "used_in_tests" | "fix_later"; /** Allows the secret through by recording a resolved alert for its fingerprint, so the next push passes. */ export async function bypassPushBlock( repoId: string, id: string, userId: string, reason: BypassReason, isAdmin: boolean, ): Promise { const db = repoDb(repoId); const row = await db.first( "SELECT * FROM secret_push_blocks WHERE repo_id = ? AND id = ?", repoId, id, ); if (!row) throw notFoundError("Blocked push not found"); if (row.user_id !== userId && !isAdmin) throw forbidden("Only the pusher or a repository admin can bypass this block"); if (reason !== "fix_later" && !isAdmin) { throw forbidden("Only repository admins can mark a secret as a false positive or test data"); } console.warn("push protection bypass", { repoId, block: id, userId, reason, detector: row.detector }); const now = Date.now(); const existing = await db.all<{ number: number }>( "SELECT number FROM secret_alerts WHERE repo_id = ? AND fingerprint = ?", repoId, row.fingerprint, ); const resolution = reason === "fix_later" ? "bypassed" : reason; const state = reason === "fix_later" ? "open" : "resolved"; const alert = existing.length > 0 ? stmt( `UPDATE secret_alerts SET state = ?, resolution = ?, resolved_by = ?, resolved_at = ?, updated_at = ? WHERE repo_id = ? AND fingerprint = ?`, state, resolution, userId, now, now, repoId, row.fingerprint, ) : stmt( `INSERT INTO secret_alerts (id, repo_id, number, detector, fingerprint, preview, state, resolution, resolved_by, resolved_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, randomId(), repoId, await nextSecurityNumber(repoId, "secret"), row.detector, row.fingerprint, row.preview, state, resolution, userId, now, now, now, ); if (row.bypassed_at !== null) throw conflict("This blocked push was already bypassed"); const [, updated] = await db.batch([ alert, stmt("UPDATE secret_push_blocks SET bypassed_at = ? WHERE id = ? AND bypassed_at IS NULL", now, id), ]); if ((updated.meta.changes ?? 0) === 0) throw conflict("This blocked push was already bypassed"); await notifyPushProtectionBypass(repoId, id, userId, reason, row.detector); }