import { env } from "cloudflare:workers"; import { credentialCoversRepo, requestCredential, scopedRepoPermission } from "./token-scope"; import type { RequestCredential, TokenArea } from "./token-scope"; import { hasPermission, loadRepo, loadRepoById, repoPermission, repoReadOnlyReason } from "./repos"; import { accountDb, repoDb } from "./scoped-db"; import { forgetLocator, locate, registerLocator } from "./scoped-db/locators"; import { scopedId } from "./scoped-db/ids"; import { attachmentUsage, syncAttachmentBytes } from "./attachment-bytes"; import { limitBrowse, limitWrite } from "./limits"; import { HttpError } from "./errors"; import { paymentRequiredResponse, planBlock } from "./metering/quota"; import type { Permission, RepoRow } from "~/lib/types"; import { contentDisposition } from "~/lib/content-disposition"; const UPLOAD_PATH = /^\/([^/]+)\/([^/]+)\/attachments$/; const SERVE_PATH = /^\/attachments\/([a-z0-9]+)\/([^/]+)$/; const MB = 1024 * 1024; const MAX_IMAGE_BYTES = 10 * MB; const MAX_FILE_BYTES = 25 * MB; const OWNER_QUOTA_BYTES = 1024 * MB; const UPLOADER_QUOTA_BYTES = 250 * MB; /** Readers without write access only ever spend their own, smaller budget; never the repository owner's. */ const GUEST_UPLOADER_QUOTA_BYTES = 50 * MB; export const ATTACHMENT_ORPHAN_GRACE_MS = 7 * 24 * 60 * 60 * 1000; const ORPHAN_GRACE_MS = ATTACHMENT_ORPHAN_GRACE_MS; const ORPHAN_SWEEP_BATCH = 50; const WIDTH_BUCKETS = [320, 640, 1280, 2560]; const MAX_IMAGE_WIDTH = WIDTH_BUCKETS[WIDTH_BUCKETS.length - 1]; const HTTP_FORBIDDEN = 403; const HTTP_TOO_LARGE = 413; const HTTP_UNSUPPORTED = 415; const TRANSFORMABLE_IMAGES = new Set(["image/png", "image/jpeg", "image/webp", "image/avif"]); const ALLOWED_TYPES = new Set([ ...TRANSFORMABLE_IMAGES, "image/gif", "video/mp4", "video/webm", "application/pdf", "application/zip", "application/gzip", "application/json", "text/plain", "text/csv", "text/markdown", ]); /** Areas whose markdown embeds attachments; a token reaches an attachment through any of them. */ const ATTACHMENT_AREAS: TokenArea[] = ["issues", "pulls", "contents"]; const WRITE_AREAS: TokenArea[] = ["issues", "pulls"]; /** Session cookie or user-content grant (full access), else a token, which stays within its grant. */ function credentialFor(request: Request): Promise { return requestCredential(request, true); } /** * The credential's permission for attachments on `repo`. A token is capped by its areas and repository list, and * a token that doesn't cover the repository only ever gets anonymous access. */ async function attachmentPermission(repo: RepoRow, credential: RequestCredential): Promise { if (!credential.grant) return repoPermission(repo, credential.viewer); let best: Permission = "none"; for (const area of ATTACHMENT_AREAS) { const permission = await scopedRepoPermission(repo, credential, area); if (hasPermission(permission, best)) best = permission; } return best; } /** Uploading with a token needs it to cover the repository and grant issue or pull request writes. */ async function tokenMayUpload(repo: RepoRow, credential: RequestCredential): Promise { const { grant } = credential; if (!grant) return true; if (grant.kind !== "fine") return false; if (!WRITE_AREAS.some((area) => grant.permissions[area] === "write")) return false; return credentialCoversRepo(credential, repo.id); } function fail(status: number, message: string): Response { return Response.json({ error: message }, { status }); } function baseType(header: string | null): string { return (header ?? "").split(";")[0].trim().toLowerCase(); } /** * Per-owner and per-uploader byte quotas. Collaborators' uploads are attributed to the repository owner * (`quotaOwnerId` = repo owner); a plain reader's uploads are attributed to the reader themself, with a * smaller budget, so throwaway accounts cannot exhaust a popular repository's storage. */ async function quotaProblem( quotaOwnerId: string, uploaderId: string, incoming: number, guest: boolean, ): Promise { const used = await attachmentUsage(quotaOwnerId, uploaderId); const uploaderCap = guest ? GUEST_UPLOADER_QUOTA_BYTES : UPLOADER_QUOTA_BYTES; if (used.uploader_total + incoming > uploaderCap) return "Your attachment storage quota is used up"; if (!guest && used.owner_total + incoming > OWNER_QUOTA_BYTES) return "The attachment storage quota for this owner is used up"; return null; } function widthBucket(requested: number): number { return WIDTH_BUCKETS.find((bucket) => bucket >= requested) ?? MAX_IMAGE_WIDTH; } async function upload(request: Request, match: RegExpExecArray): Promise { if (request.method !== "POST") return fail(405, "Method not allowed"); if (request.headers.get("x-gitfor-csrf") !== "1" && !request.headers.get("authorization")) return fail(HTTP_FORBIDDEN, "Missing CSRF header"); const credential = await credentialFor(request); const viewer = credential.viewer; if (!viewer) return fail(401, "Sign in to upload files"); await limitWrite(viewer); const repo = await loadRepo(match[1], decodeURIComponent(match[2])); const permission = repo ? await attachmentPermission(repo, credential) : "none"; if (!repo || permission === "none") return fail(404, "Not found"); if (!(await tokenMayUpload(repo, credential))) return fail(HTTP_FORBIDDEN, "This token cannot upload attachments"); const readOnly = repoReadOnlyReason(repo); if (readOnly) return fail(HTTP_FORBIDDEN, readOnly); const guest = permission === "read"; const quotaOwnerId = guest ? viewer.id : repo.owner_id; const contentType = baseType(request.headers.get("content-type")); if (!ALLOWED_TYPES.has(contentType)) return fail(HTTP_UNSUPPORTED, `Files of type ${contentType} are not allowed`); const size = Number(request.headers.get("content-length") ?? "0"); const cap = contentType.startsWith("image/") ? MAX_IMAGE_BYTES : MAX_FILE_BYTES; if (!request.body || !Number.isFinite(size) || size <= 0) return fail(400, "A non-empty body is required"); if (size > cap) return fail(HTTP_TOO_LARGE, `Files of this type are limited to ${cap / MB} MB`); const overQuota = await quotaProblem(quotaOwnerId, viewer.id, size, guest); if (overQuota) return fail(HTTP_TOO_LARGE, overQuota); const overPlan = await planBlock(quotaOwnerId, "storage.r2", size); if (overPlan) return await paymentRequiredResponse(overPlan, viewer.id); const rawName = new URL(request.url).searchParams.get("name") ?? "file"; const name = rawName.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 100) || "file"; const id = scopedId(repo.id); const key = `attachments/${quotaOwnerId}/${id}`; const stored = await env.BUCKET.put(key, request.body.pipeThrough(new FixedLengthStream(size)), { httpMetadata: { contentType }, }); const db = repoDb(repo.id); await db.run( `INSERT INTO attachments (id, owner_id, repo_id, uploader_id, name, content_type, size, r2_key, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, id, quotaOwnerId, repo.id, viewer.id, name, contentType, stored.size, key, Date.now(), ); await registerLocator("attachment", id, repo.id); await syncAttachmentBytes(repo.id, [quotaOwnerId, viewer.id]); if (await quotaProblem(quotaOwnerId, viewer.id, 0, guest)) { await Promise.all([db.run("DELETE FROM attachments WHERE id = ?", id), env.BUCKET.delete(key)]); await forgetLocator("attachment", id); await syncAttachmentBytes(repo.id, [quotaOwnerId, viewer.id]); return fail(HTTP_TOO_LARGE, "Attachment storage quota exceeded"); } const url = `/attachments/${id}/${encodeURIComponent(name)}`; const markdown = contentType.startsWith("image/") ? `![${name}](${url})` : `[${name}](${url})`; return Response.json({ id, url, markdown }); } async function serve(request: Request, match: RegExpExecArray): Promise { const credential = await credentialFor(request); await limitBrowse(credential.viewer, request); const repoId = await locate("attachment", match[1]); const attachment = repoId ? await repoDb(repoId).first<{ repo_id: string; r2_key: string; content_type: string; name: string; }>("SELECT repo_id, r2_key, content_type, name FROM attachments WHERE id = ?", match[1]) : null; if (!attachment) return new Response("Not found", { status: 404 }); const repo = await loadRepoById(attachment.repo_id); if (!repo || (await attachmentPermission(repo, credential)) === "none") return new Response("Not found", { status: 404 }); const object = await env.BUCKET.get(attachment.r2_key); if (!object) return new Response("Attachment missing", { status: 410 }); const cache = repo.visibility === "public" ? "public, max-age=3600" : "private, max-age=3600"; const headers = { "content-security-policy": "sandbox; default-src 'none'", "x-content-type-options": "nosniff", "cache-control": cache, }; if (TRANSFORMABLE_IMAGES.has(attachment.content_type)) { const requested = Number(new URL(request.url).searchParams.get("w") ?? MAX_IMAGE_WIDTH); const width = widthBucket(Number.isFinite(requested) && requested > 0 ? requested : MAX_IMAGE_WIDTH); const result = await env.IMAGES.input(object.body) .transform({ width, fit: "scale-down" }) .output({ format: "image/webp" }); return result.response({ headers }); } const inline = attachment.content_type.startsWith("image/") || attachment.content_type.startsWith("video/"); return new Response(object.body, { headers: { ...headers, "content-type": attachment.content_type, "content-length": String(object.size), "content-disposition": contentDisposition(inline ? "inline" : "attachment", attachment.name), }, }); } /** Markdown attachment upload (`POST ///attachments`) and download (`GET /attachments//`). */ export async function handleAttachmentRoutes(request: Request, url: URL): Promise { const uploadMatch = UPLOAD_PATH.exec(url.pathname); try { if (uploadMatch) return await upload(request, uploadMatch); const serveMatch = SERVE_PATH.exec(url.pathname); if (serveMatch && (request.method === "GET" || request.method === "HEAD")) return await serve(request, serveMatch); } catch (error) { if (error instanceof HttpError) return fail(error.status, error.message); throw error; } return null; } /** Every repository column that stores user markdown; an attachment referenced from none of them after the grace period is garbage. */ const MARKDOWN_SOURCES = [ "SELECT 1 FROM issues WHERE body LIKE ?1", "SELECT 1 FROM comments WHERE body LIKE ?1", "SELECT 1 FROM review_comments WHERE body LIKE ?1", "SELECT 1 FROM reviews WHERE body LIKE ?1", "SELECT 1 FROM discussions WHERE body LIKE ?1", "SELECT 1 FROM discussion_comments WHERE body LIKE ?1", "SELECT 1 FROM releases WHERE body LIKE ?1", "SELECT 1 FROM milestones WHERE description LIKE ?1", "SELECT 1 FROM security_advisories WHERE description LIKE ?1", "SELECT 1 FROM advisory_comments WHERE body LIKE ?1", ]; type Candidate = { id: string; r2_key: string; owner_id: string; uploader_id: string }; /** Looks in the attachment's own repository, and in the projects of its quota owner and of the repository's owner. */ async function isReferenced(repo: { id: string; owner_id: string }, candidate: Candidate): Promise { const pattern = `%/attachments/${candidate.id}/%`; for (const sql of MARKDOWN_SOURCES) { if ((await repoDb(repo.id).first(`${sql} LIMIT 1`, pattern)) !== null) return true; } for (const accountId of new Set([repo.owner_id, candidate.owner_id])) { const project = await accountDb(accountId).first( "SELECT 1 FROM projects WHERE description LIKE ?1 LIMIT 1", pattern, ); if (project !== null) return true; } return false; } /** * Deletes the repository's attachments uploaded over a week ago that no issue, comment, release or advisory body * links to. Bounded per run; run by the RepoState alarm when the oldest attachment passes the grace period. */ export async function sweepOrphanedAttachmentsForRepo(repoId: string, now: number): Promise { const repo = await loadRepoById(repoId); // Its directory row is gone (deletion in flight or failed): nothing to check references against; try next hour. if (!repo) { console.warn("Attachment sweep skipped: repository has no directory row", repoId); return false; } const db = repoDb(repo.id); const candidates = await db.all( "SELECT id, r2_key, owner_id, uploader_id FROM attachments WHERE created_at < ? ORDER BY created_at LIMIT ?", now - ORPHAN_GRACE_MS, ORPHAN_SWEEP_BATCH, ); for (const candidate of candidates) { if (await isReferenced(repo, candidate)) { // Touch so the next sweep looks at other old rows first instead of re-scanning the same referenced ones. await db.run("UPDATE attachments SET created_at = ? WHERE id = ?", now, candidate.id); continue; } await env.BUCKET.delete(candidate.r2_key); await db.run("DELETE FROM attachments WHERE id = ?", candidate.id); await forgetLocator("attachment", candidate.id); await syncAttachmentBytes(repo.id, [candidate.owner_id, candidate.uploader_id]); } return candidates.length === ORPHAN_SWEEP_BATCH; }