import { all, batch, chunks, MAX_IN_LIST, placeholders, stmt } from "../db"; import { loadRepoById } from "../repos"; import { withStore } from "../git/store"; import { manifestKind } from "./manifests"; import type { Dependency, Ecosystem } from "./manifests"; import { loadSecuritySettings } from "./settings"; const MAX_TREE_ENTRIES = 20_000; const MAX_MANIFESTS = 50; const MAX_MANIFEST_BYTES = 2 * 1024 * 1024; const MAX_DEPENDENCIES = 5000; const DEPENDENCY_COLUMNS = 7; const STATEMENTS_PER_BATCH = 50; const DEPENDENCY_PAGE = 100; const MAX_NAME_LENGTH = 214; const decoder = new TextDecoder(); type Parsed = { path: string; ecosystem: Ecosystem; dependencies: Dependency[] }; /** Rebuilds the dependency graph for a commit from its manifests and lockfiles, within fixed budgets. */ export async function scanDependencies(repoId: string, sha: string): Promise { const repo = await loadRepoById(repoId); if (!repo || !(await loadSecuritySettings(repo.id)).dependencyGraph) return 0; const parsed = await withStore(repo.artifacts_name, async (store) => { const commit = await store.commit(sha); if (!commit) return null; const { entries } = await store.walk(commit.treeSha, MAX_TREE_ENTRIES); const manifests = entries .filter((entry) => entry.type === "blob" && manifestKind(entry.path)) .slice(0, MAX_MANIFESTS); const out: Parsed[] = []; let total = 0; for (const entry of manifests) { const kind = manifestKind(entry.path); const blob = kind ? await store.blob(entry.sha) : null; if (!kind || !blob || blob.byteLength > MAX_MANIFEST_BYTES) continue; const unique = new Map(); for (const dependency of kind.parse(decoder.decode(blob))) { const key = `${dependency.name}\0${dependency.scope}`; if (!unique.has(key) && dependency.name.length <= MAX_NAME_LENGTH) unique.set(key, dependency); } const dependencies = [...unique.values()].slice(0, MAX_DEPENDENCIES - total); total += dependencies.length; out.push({ path: entry.path, ecosystem: kind.ecosystem, dependencies }); if (total >= MAX_DEPENDENCIES) break; } return out; }); if (!parsed) return 0; const now = Date.now(); const statements: D1PreparedStatement[] = [ stmt("DELETE FROM dependencies WHERE repo_id = ?", repo.id), stmt("DELETE FROM dependency_manifests WHERE repo_id = ?", repo.id), ]; for (const manifest of parsed) { statements.push( stmt( `INSERT INTO dependency_manifests (repo_id, path, ecosystem, commit_sha, dependency_count, scanned_at) VALUES (?, ?, ?, ?, ?, ?)`, repo.id, manifest.path, manifest.ecosystem, sha, manifest.dependencies.length, now, ), ); for (const part of chunks(manifest.dependencies, Math.floor(MAX_IN_LIST / DEPENDENCY_COLUMNS))) { statements.push( stmt( `INSERT OR IGNORE INTO dependencies (repo_id, manifest_path, ecosystem, name, requirement, scope, direct) VALUES ${part.map(() => `(${placeholders(DEPENDENCY_COLUMNS)})`).join(", ")}`, ...part.flatMap((dependency) => [ repo.id, manifest.path, manifest.ecosystem, dependency.name, dependency.requirement, dependency.scope, dependency.direct ? 1 : 0, ]), ), ); } } for (const group of chunks(statements, STATEMENTS_PER_BATCH)) await batch(group); return parsed.reduce((sum, manifest) => sum + manifest.dependencies.length, 0); } export type DependencyRow = { manifest_path: string; ecosystem: Ecosystem; name: string; requirement: string | null; scope: "runtime" | "development"; direct: number; advisories: number; }; export type ManifestRow = { path: string; ecosystem: Ecosystem; commit_sha: string; dependency_count: number; scanned_at: number; }; export async function listDependencies( repoId: string, filter: { query: string; ecosystem: Ecosystem | null; page: number }, ): Promise<{ manifests: ManifestRow[]; dependencies: DependencyRow[]; hasMore: boolean }> { const manifests = await all( "SELECT path, ecosystem, commit_sha, dependency_count, scanned_at FROM dependency_manifests WHERE repo_id = ? ORDER BY path", repoId, ); const rows = await all( `SELECT d.manifest_path, d.ecosystem, d.name, d.requirement, d.scope, d.direct, (SELECT COUNT(*) FROM security_advisories sa WHERE sa.state = 'published' AND sa.ecosystem = d.ecosystem AND sa.package_name = d.name) AS advisories FROM dependencies d WHERE d.repo_id = ?1 AND (?2 = '' OR d.name LIKE '%' || ?2 || '%') AND (?3 = '' OR d.ecosystem = ?3) ORDER BY d.direct DESC, d.name LIMIT ?4 OFFSET ?5`, repoId, filter.query.replace(/[%_]/g, ""), filter.ecosystem ?? "", DEPENDENCY_PAGE + 1, (filter.page - 1) * DEPENDENCY_PAGE, ); return { manifests, dependencies: rows.slice(0, DEPENDENCY_PAGE), hasMore: rows.length > DEPENDENCY_PAGE }; }