import { all, run } from "../db"; import { accountDb } from "../scoped-db"; import { loadRepoById } from "../repos"; import { notifyQuietly } from "./deliver"; import { accountAdmins, repoAdmins } from "./recipients"; const OFFLINE_AFTER_MS = 10 * 60 * 1000; /** Runners silent for longer than this are considered abandoned and no longer reported. */ const FORGET_AFTER_MS = 24 * 60 * 60 * 1000; const RUNNERS_PER_SWEEP = 50; type RunnerRow = { id: string; name: string; repo_id: string | null; owner_id: string | null; last_seen_at: number }; async function notifyRunnerOffline(runner: RunnerRow): Promise { const minutes = Math.round((Date.now() - runner.last_seen_at) / 60_000); const details = [ `Runner "${runner.name}" has not checked in for ${minutes} minutes.`, "Queued jobs for it will wait.", ]; if (runner.repo_id) { const repo = await loadRepoById(runner.repo_id); if (!repo) return; await notifyQuietly({ category: "runner", repoId: repo.id, accountId: null, access: "admin", recipients: await repoAdmins(repo.id), title: `Runner ${runner.name} is offline (${repo.owner_name}/${repo.name})`, details, url: `/${repo.owner_name}/${repo.name}/settings/runners`, }); return; } if (!runner.owner_id) return; await notifyQuietly({ category: "runner", repoId: null, accountId: runner.owner_id, access: "account", recipients: await accountAdmins(runner.owner_id), title: `Runner ${runner.name} is offline`, details, url: "/settings/actions", }); } /** Cron: reports each runner once per outage, and clears the mark when it checks in again. */ export async function sweepOfflineRunners(): Promise { const now = Date.now(); const notices = await all<{ runner_id: string; notified_at: number }>( "SELECT runner_id, notified_at FROM runner_offline_notices", ); const noticed = new Map(notices.map((notice) => [notice.runner_id, notice.notified_at])); const offline: RunnerRow[] = []; for (const account of await all<{ id: string }>("SELECT id FROM accounts")) { const runners = await accountDb(account.id).all( "SELECT id, name, repo_id, owner_id, last_seen_at FROM runners WHERE last_seen_at IS NOT NULL", ); for (const runner of runners) { const notifiedAt = noticed.get(runner.id); if (notifiedAt !== undefined && runner.last_seen_at > notifiedAt) { await run("DELETE FROM runner_offline_notices WHERE runner_id = ?", runner.id); noticed.delete(runner.id); } if ( runner.last_seen_at < now - OFFLINE_AFTER_MS && runner.last_seen_at > now - FORGET_AFTER_MS && !noticed.has(runner.id) && offline.length < RUNNERS_PER_SWEEP ) offline.push(runner); } } for (const runner of offline) { const claimed = await run( "INSERT OR IGNORE INTO runner_offline_notices (runner_id, notified_at) VALUES (?, ?)", runner.id, now, ); if (claimed.meta.changes === 0) continue; await notifyRunnerOffline(runner).catch((error: unknown) => console.error("Runner offline notice failed", error)); } }