import { HttpError } from "../errors"; import { isArtifactsError } from "./artifacts"; import { recordArtifactsCall, recordArtifactsFailure } from "./artifacts-health"; const HTTP_BAD_GATEWAY = 502; const REFERENCE = /reference\s*=\s*([a-z0-9]+)/i; const RETRY_BASE_MS = 150; const MAX_ATTEMPTS = 3; /** Concurrent binding calls per limiter (one per request's store); bursts are what trip opaque errors. */ const MAX_IN_FLIGHT = 6; /** A hung binding call must not hold a limiter slot forever. */ const CALL_TIMEOUT_MS = 30_000; const REMOTE_PREFIX = /^ArtifactsError:\s*/; const CODE_PATTERNS: Array<[RegExp, ArtifactsErrorCode]> = [ [/being forked|fork is still in progress/i, "FORK_IN_PROGRESS"], [/being imported|import is still in progress/i, "IMPORT_IN_PROGRESS"], [/being created/i, "CREATE_IN_PROGRESS"], [/already exists/i, "ALREADY_EXISTS"], [/requires? auth/i, "REMOTE_AUTH_REQUIRED"], [/memory limit/i, "MEMORY_LIMIT"], [/not found|does not exist/i, "NOT_FOUND"], ]; /** * Remote bindings in local dev (and some RPC paths) deliver ArtifactsErrors as plain Errors whose message * starts with "ArtifactsError:", losing `name` and `code`; rebuild a typed error so callers can branch on it. */ function revive(error: unknown): ArtifactsError | null { if (!(error instanceof Error) || !REMOTE_PREFIX.test(error.message)) return null; const message = error.message.replace(REMOTE_PREFIX, ""); const code = CODE_PATTERNS.find(([pattern]) => pattern.test(message))?.[1] ?? "INTERNAL_ERROR"; return Object.assign(new Error(message), { name: "ArtifactsError" as const, code, numericCode: 0 }); } /** An Artifacts failure without a typed error code, e.g. the opaque "internal error; reference = …". */ export class ArtifactsUnavailableError extends HttpError { readonly operation: string; readonly reference: string | null; constructor(operation: string, target: string, cause: unknown) { const detail = cause instanceof Error ? `${cause.message} ${cause.stack ?? ""} ${String(cause.cause ?? "")}` : String(cause); const reference = REFERENCE.exec(detail)?.[1] ?? null; super( HTTP_BAD_GATEWAY, `Cloudflare Artifacts failed to ${operation} for ${target}${reference ? ` (reference ${reference})` : ""}. ` + "This is usually transient; try again shortly.", ); this.name = "ArtifactsUnavailableError"; this.operation = operation; this.reference = reference; } } /** * Runs an Artifacts binding call, keeping typed ArtifactsErrors intact (callers branch on their codes) * but turning untyped failures into a logged, user-readable error that names the operation and repo. * Idempotent reads are retried once, since untyped failures are usually transient. */ /** * Caps concurrent binding calls. Scoped to one request (each GitStore owns one) because promises * resolved across Workers request contexts can hang; never share an instance between requests. */ export class CallLimiter { private inFlight = 0; private readonly waiting: Array<() => void> = []; async run(call: () => Promise): Promise { if (this.inFlight < MAX_IN_FLIGHT) this.inFlight++; else await new Promise((resolve) => this.waiting.push(resolve)); try { return await withTimeout(call()); } finally { const next = this.waiting.shift(); if (next) next(); else this.inFlight--; } } } class CallTimeoutError extends Error {} function isDisposable(value: unknown): value is Disposable { return typeof value === "object" && value !== null && Symbol.dispose in value; } /** * A call that is abandoned on timeout may still resolve later with an RPC stub (an `ArtifactsRepo` * handle); nobody is left to dispose it, so release it here instead of letting the runtime warn. */ async function withTimeout(promise: Promise): Promise { let timer: ReturnType | undefined; let timedOut = false; const timeout = new Promise((_, reject) => { timer = setTimeout(() => { timedOut = true; reject(new CallTimeoutError("Artifacts call timed out")); }, CALL_TIMEOUT_MS); }); promise.then( (value) => { if (timedOut && isDisposable(value)) value[Symbol.dispose](); }, () => undefined, ); try { return await Promise.race([promise, timeout]); } finally { clearTimeout(timer); } } function backoff(attempt: number): Promise { const delay = RETRY_BASE_MS * 2 ** (attempt - 1) * (0.5 + Math.random()); return new Promise((resolve) => setTimeout(resolve, delay)); } /** * Runs an Artifacts binding call (under the caller's request-scoped limiter, with a timeout), keeping typed ArtifactsErrors intact * (callers branch on their codes) but turning untyped failures into a logged, user-readable error that * names the operation and repo. Idempotent reads retry untyped failures with jittered exponential backoff. */ export async function artifactsCall( operation: string, target: string, call: () => Promise, options: { idempotent?: boolean; limiter?: CallLimiter } = {}, ): Promise { const attempts = options.idempotent ? MAX_ATTEMPTS : 1; for (let attempt = 1; ; attempt++) { try { const result = await (options.limiter ? options.limiter.run(call) : withTimeout(call())); recordArtifactsCall(operation); return result; } catch (error) { if (typedFailure(error)) recordArtifactsCall(operation); if (attempt >= attempts || typedFailure(error)) return rethrow(operation, target, error, attempt); await backoff(attempt); } } } /** Everything a remote-binding error carries, since its message alone is often empty or opaque. */ function errorDetail(error: unknown): Record { if (!(error instanceof Error)) return { value: String(error) }; const extra = Object.fromEntries(Object.entries(error)); return { name: error.name, message: error.message, stack: error.stack, cause: error.cause, ...extra }; } function typedFailure(error: unknown): boolean { if (isArtifactsError(error) || error instanceof ArtifactsUnavailableError) return true; const revived = revive(error); return revived !== null && revived.code !== "INTERNAL_ERROR"; } function rethrow(operation: string, target: string, error: unknown, attempts: number): never { if (isArtifactsError(error) || error instanceof ArtifactsUnavailableError) throw error; const revived = revive(error); if (revived && revived.code !== "INTERNAL_ERROR") throw revived; const wrapped = new ArtifactsUnavailableError(operation, target, error); console.error(`Artifacts ${operation} failed`, { target, reference: wrapped.reference, error: errorDetail(error) }); recordArtifactsFailure({ operation, target, reference: wrapped.reference, message: error instanceof Error ? error.message : String(error), attempts, }); throw wrapped; }