import { all, run } from "../db"; import { loadRepoById } from "../repos"; import { notifyQuietly } from "./deliver"; import { accountAdmins, repoAdmins } from "./recipients"; const OFFLINE_AFTER_MS = 10 * 60 * 1000; /** Runners silent for longer than this are considered abandoned and no longer reported. */ const FORGET_AFTER_MS = 24 * 60 * 60 * 1000; const RUNNERS_PER_SWEEP = 50; type RunnerRow = { id: string; name: string; repo_id: string | null; owner_id: string | null; last_seen_at: number }; async function notifyRunnerOffline(runner: RunnerRow): Promise { const minutes = Math.round((Date.now() - runner.last_seen_at) / 60_000); const details = [ `Runner "${runner.name}" has not checked in for ${minutes} minutes.`, "Queued jobs for it will wait.", ]; if (runner.repo_id) { const repo = await loadRepoById(runner.repo_id); if (!repo) return; await notifyQuietly({ category: "runner", repoId: repo.id, accountId: null, access: "admin", recipients: await repoAdmins(repo.id), title: `Runner ${runner.name} is offline (${repo.owner_name}/${repo.name})`, details, url: `/${repo.owner_name}/${repo.name}/settings/runners`, }); return; } if (!runner.owner_id) return; await notifyQuietly({ category: "runner", repoId: null, accountId: runner.owner_id, access: "account", recipients: await accountAdmins(runner.owner_id), title: `Runner ${runner.name} is offline`, details, url: "/settings/actions", }); } /** Cron: reports each runner once per outage, and clears the mark when it checks in again. */ export async function sweepOfflineRunners(): Promise { const now = Date.now(); await run( `DELETE FROM runner_offline_notices WHERE runner_id IN ( SELECT n.runner_id FROM runner_offline_notices n JOIN runners r ON r.id = n.runner_id WHERE r.last_seen_at > n.notified_at)`, ); const offline = await all( `SELECT r.id, r.name, r.repo_id, r.owner_id, r.last_seen_at FROM runners r WHERE r.last_seen_at IS NOT NULL AND r.last_seen_at < ? AND r.last_seen_at > ? AND NOT EXISTS (SELECT 1 FROM runner_offline_notices n WHERE n.runner_id = r.id) LIMIT ?`, now - OFFLINE_AFTER_MS, now - FORGET_AFTER_MS, RUNNERS_PER_SWEEP, ); for (const runner of offline) { const claimed = await run( "INSERT OR IGNORE INTO runner_offline_notices (runner_id, notified_at) VALUES (?, ?)", runner.id, now, ); if (claimed.meta.changes === 0) continue; await notifyRunnerOffline(runner).catch((error: unknown) => console.error("Runner offline notice failed", error)); } }