import { chunks, MAX_IN_LIST, placeholders } from "../db"; import { repoDb, stmt, type ScopedStatement } from "../scoped-db"; import { packageKey, publishedAdvisoryCounts } from "./advisory-packages"; import { loadRepoById } from "../repos"; import { withStore } from "../git/store"; import { branchSha, refreshRefs } from "../git/refs"; import { manifestKind } from "./manifests"; import type { Dependency, Ecosystem } from "./manifests"; import { notifyDependencyAlerts } from "../notify/security"; import { loadSecuritySettings } from "./settings"; const MAX_TREE_ENTRIES = 20_000; const MAX_MANIFESTS = 50; const MAX_MANIFEST_BYTES = 2 * 1024 * 1024; const MAX_DEPENDENCIES = 5000; const DEPENDENCY_COLUMNS = 7; const STATEMENTS_PER_BATCH = 50; const DEPENDENCY_PAGE = 100; const MAX_NAME_LENGTH = 214; const MAX_REQUIREMENT_LENGTH = 100; const decoder = new TextDecoder(); type Parsed = { path: string; ecosystem: Ecosystem; dependencies: Dependency[] }; /** Rebuilds the dependency graph for a commit from its manifests and lockfiles, within fixed budgets. */ export async function scanDependencies(repoId: string, sha: string): Promise { const repo = await loadRepoById(repoId); if (!repo || !(await loadSecuritySettings(repo.id)).dependencyGraph) return 0; const parsed = await withStore(repo.artifacts_name, async (store) => { const commit = await store.commit(sha); if (!commit) return null; const { entries } = await store.walk(commit.treeSha, MAX_TREE_ENTRIES); const manifests = entries .filter((entry) => entry.type === "blob" && manifestKind(entry.path)) .slice(0, MAX_MANIFESTS); const out: Parsed[] = []; let total = 0; for (const entry of manifests) { const kind = manifestKind(entry.path); const blob = kind ? await store.blob(entry.sha) : null; if (!kind || !blob || blob.byteLength > MAX_MANIFEST_BYTES) continue; const unique = new Map(); for (const dependency of kind.parse(decoder.decode(blob))) { const key = `${dependency.name}\0${dependency.scope}`; if (!unique.has(key) && dependency.name.length <= MAX_NAME_LENGTH) { unique.set(key, { ...dependency, requirement: dependency.requirement === null ? null : dependency.requirement.slice(0, MAX_REQUIREMENT_LENGTH), }); } } const dependencies = [...unique.values()].slice(0, MAX_DEPENDENCIES - total); total += dependencies.length; out.push({ path: entry.path, ecosystem: kind.ecosystem, dependencies }); if (total >= MAX_DEPENDENCIES) break; } return out; }); if (!parsed) return 0; // Scans are queued per push and can finish out of order; only the current default-branch head may // replace the graph, so a stale scan of an older commit can't wipe newer results. const head = branchSha(await refreshRefs(repo), repo.default_branch); if (head !== sha) return 0; const vulnerableBefore = new Set(await vulnerablePackages(repo.id)); const now = Date.now(); const statements: ScopedStatement[] = [ stmt("DELETE FROM dependencies WHERE repo_id = ?", repo.id), stmt("DELETE FROM dependency_manifests WHERE repo_id = ?", repo.id), ]; for (const manifest of parsed) { statements.push( stmt( `INSERT INTO dependency_manifests (repo_id, path, ecosystem, commit_sha, dependency_count, scanned_at) VALUES (?, ?, ?, ?, ?, ?)`, repo.id, manifest.path, manifest.ecosystem, sha, manifest.dependencies.length, now, ), ); for (const part of chunks(manifest.dependencies, Math.floor(MAX_IN_LIST / DEPENDENCY_COLUMNS))) { statements.push( stmt( `INSERT OR IGNORE INTO dependencies (repo_id, manifest_path, ecosystem, name, requirement, scope, direct) VALUES ${part.map(() => `(${placeholders(DEPENDENCY_COLUMNS)})`).join(", ")}`, ...part.flatMap((dependency) => [ repo.id, manifest.path, manifest.ecosystem, dependency.name, dependency.requirement, dependency.scope, dependency.direct ? 1 : 0, ]), ), ); } } for (const group of chunks(statements, STATEMENTS_PER_BATCH)) await repoDb(repo.id).batch(group); const vulnerableAfter = await vulnerablePackages(repo.id); if (vulnerableBefore.size < MAX_VULNERABLE_LISTED && vulnerableAfter.length < MAX_VULNERABLE_LISTED) { await notifyDependencyAlerts( repo.id, vulnerableAfter.filter((name) => !vulnerableBefore.has(name)), ); } else { console.warn("Too many vulnerable dependencies to diff; skipping dependency alert", repo.id); } return parsed.reduce((sum, manifest) => sum + manifest.dependencies.length, 0); } const MAX_VULNERABLE_LISTED = 200; /** Distinct `ecosystem:name` dependencies matching a published advisory, bounded. */ async function vulnerablePackages(repoId: string): Promise { const rows = await repoDb(repoId).all<{ ecosystem: string; name: string }>( "SELECT DISTINCT ecosystem, name FROM dependencies WHERE repo_id = ? ORDER BY ecosystem, name", repoId, ); if (rows.length === 0) return []; const published = await publishedAdvisoryCounts(rows.map((row) => ({ ecosystem: row.ecosystem, name: row.name }))); return rows .filter((row) => published.has(packageKey(row.ecosystem, row.name))) .slice(0, MAX_VULNERABLE_LISTED) .map((row) => `${row.ecosystem}:${row.name}`); } export type DependencyRow = { manifest_path: string; ecosystem: Ecosystem; name: string; requirement: string | null; scope: "runtime" | "development"; direct: number; advisories: number; }; export type ManifestRow = { path: string; ecosystem: Ecosystem; commit_sha: string; dependency_count: number; scanned_at: number; }; export async function listDependencies( repoId: string, filter: { query: string; ecosystem: Ecosystem | null; page: number }, ): Promise<{ manifests: ManifestRow[]; dependencies: DependencyRow[]; hasMore: boolean }> { const db = repoDb(repoId); const manifests = await db.all( "SELECT path, ecosystem, commit_sha, dependency_count, scanned_at FROM dependency_manifests WHERE repo_id = ? ORDER BY path", repoId, ); const stored = await db.all>( `SELECT d.manifest_path, d.ecosystem, d.name, d.requirement, d.scope, d.direct FROM dependencies d WHERE d.repo_id = ?1 AND (?2 = '' OR d.name LIKE '%' || ?2 || '%') AND (?3 = '' OR d.ecosystem = ?3) ORDER BY d.direct DESC, d.name LIMIT ?4 OFFSET ?5`, repoId, filter.query.replace(/[%_]/g, ""), filter.ecosystem ?? "", DEPENDENCY_PAGE + 1, (filter.page - 1) * DEPENDENCY_PAGE, ); const published = await publishedAdvisoryCounts(stored.map((row) => ({ ecosystem: row.ecosystem, name: row.name }))); const rows: DependencyRow[] = stored.map((row) => ({ ...row, advisories: published.get(packageKey(row.ecosystem, row.name)) ?? 0, })); return { manifests, dependencies: rows.slice(0, DEPENDENCY_PAGE), hasMore: rows.length > DEPENDENCY_PAGE }; }