import { loadRepoById } from "../repos"; import { diffTreesBounded } from "../git/diff"; import { ZERO_SHA, withStore } from "../git/store"; import type { GitStore } from "../git/store"; import { listRefs } from "../git/refs"; import { compareCommits } from "../git/history"; import { redactFindings, scannableText, scanText } from "./detectors"; import { recordSecretAlerts } from "./secret-alerts"; import type { LocatedFinding } from "./secret-alerts"; import { loadSecuritySettings } from "./settings"; const MAX_SCAN_FILES = 1000; const MAX_SCAN_COMMITS = 50; const RISKY_NAMES = /(^|\/)(\.env[^/]*|[^/]*\.(pem|key|p12|pfx|json|ya?ml|toml|ini|cfg|conf|properties|tf|tfvars|sh|py|js|ts|rb|go)|[^/]*(secret|credential|token|password|config)[^/]*)$/i; const MAX_BLOB_BYTES = 1024 * 1024; const MAX_TOTAL_BYTES = 16 * 1024 * 1024; const BLOB_CONCURRENCY = 8; const SKIPPED_EXTENSIONS = /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|jar|woff2?|ttf|otf|mp[34]|mov|wasm)$/i; type Candidate = { path: string; sha: string }; async function candidatesFor(store: GitStore, baseTree: string | null, headTree: string): Promise { if (baseTree) { const { changes } = await diffTreesBounded(store, baseTree, headTree); return changes .filter((change) => change.newSha && change.status !== "deleted") .map((change) => ({ path: change.path, sha: change.newSha ?? "" })); } const { entries } = await store.walk(headTree, MAX_SCAN_FILES * 4); return entries.filter((entry) => entry.type === "blob").map((entry) => ({ path: entry.path, sha: entry.sha })); } /** * Blobs introduced by every commit in `base..head` (bounded), not just the tip diff, so a secret added and * removed within one push is still seen. Deduplicated by blob, with likely secret-bearing paths first. */ async function pushCandidates(store: GitStore, baseSha: string | null, headSha: string, headTree: string) { const commits = baseSha ? (await compareCommits(store, baseSha, store, headSha)).commits.slice(0, MAX_SCAN_COMMITS) : []; const byBlob = new Map(); const add = (candidates: Candidate[], commitSha: string) => { for (const candidate of candidates) { if (!byBlob.has(candidate.sha)) byBlob.set(candidate.sha, { ...candidate, commitSha }); } }; if (!baseSha) add(await candidatesFor(store, null, headTree), headSha); for (const commit of commits) { const parent = commit.parents[0] ? await store.commit(commit.parents[0]) : null; add(await candidatesFor(store, parent?.treeSha ?? null, commit.treeSha), commit.sha); } const ranked = [...byBlob.values()].filter((candidate) => !SKIPPED_EXTENSIONS.test(candidate.path)); ranked.sort((left, right) => Number(RISKY_NAMES.test(right.path)) - Number(RISKY_NAMES.test(left.path))); return ranked.slice(0, MAX_SCAN_FILES); } /** Scans blobs introduced by a push (bounded by file count and bytes) and records secret alerts. */ export async function scanPushForSecrets(repoId: string, before: string, after: string): Promise { if (after === ZERO_SHA) return 0; const repo = await loadRepoById(repoId); if (!repo) return 0; if (!(await loadSecuritySettings(repo.id)).secretScanning) return 0; const refs = await listRefs(repo); const defaultSha = refs.branches.find((branch) => branch.name === repo.default_branch)?.sha ?? null; const findings = await withStore(repo.artifacts_name, async (store) => { const head = await store.commit(after); if (!head) return []; const baseSha = before !== ZERO_SHA ? before : defaultSha && defaultSha !== after ? defaultSha : null; const base = baseSha ? await store.commit(baseSha) : null; const candidates = await pushCandidates(store, base ? baseSha : null, after, head.treeSha); const out: LocatedFinding[] = []; let total = 0; for (let index = 0; index < candidates.length && total < MAX_TOTAL_BYTES; index += BLOB_CONCURRENCY) { const group = candidates.slice(index, index + BLOB_CONCURRENCY); const blobs = await Promise.all(group.map((candidate) => store.blob(candidate.sha))); for (const [position, blob] of blobs.entries()) { if (!blob || blob.byteLength > MAX_BLOB_BYTES) continue; total += blob.byteLength; const text = scannableText(blob); if (!text) continue; for (const finding of await redactFindings(scanText(text))) { out.push({ ...finding, path: group[position].path, commitSha: group[position].commitSha }); } } } return out; }); if (findings.length === 0) return 0; return recordSecretAlerts(repo.id, findings); }