import { loadRepoById } from "../repos"; import { notifyQuietly } from "./deliver"; import { accountAdmins, repoAdmins } from "./recipients"; const OFFLINE_AFTER_MS = 10 * 60 * 1000; /** Runners silent for longer than this are considered abandoned and no longer reported. */ const FORGET_AFTER_MS = 24 * 60 * 60 * 1000; const RUNNERS_PER_SWEEP = 50; type RunnerRow = { id: string; name: string; repo_id: string | null; owner_id: string | null; last_seen_at: number }; async function notifyRunnerOffline(runner: RunnerRow): Promise { const minutes = Math.round((Date.now() - runner.last_seen_at) / 60_000); const details = [ `Runner "${runner.name}" has not checked in for ${minutes} minutes.`, "Queued jobs for it will wait.", ]; if (runner.repo_id) { const repo = await loadRepoById(runner.repo_id); if (!repo) return; await notifyQuietly({ category: "runner", repoId: repo.id, accountId: null, access: "admin", recipients: await repoAdmins(repo.id), title: `Runner ${runner.name} is offline (${repo.owner_name}/${repo.name})`, details, url: `/${repo.owner_name}/${repo.name}/settings/runners`, }); return; } if (!runner.owner_id) return; await notifyQuietly({ category: "runner", repoId: null, accountId: runner.owner_id, access: "account", recipients: await accountAdmins(runner.owner_id), title: `Runner ${runner.name} is offline`, details, url: "/settings/actions", }); } type Sql = { exec>( sql: string, ...params: Array ): { toArray(): T[] }; }; /** When the account's next runner crosses the offline threshold without having been reported, or null. */ export function nextRunnerOffline(sql: Sql, now: number): number | null { const row = sql .exec<{ due: number | null }>( `SELECT MIN(r.last_seen_at) + ${OFFLINE_AFTER_MS} AS due FROM runners r WHERE r.last_seen_at IS NOT NULL AND r.last_seen_at > ? - ${FORGET_AFTER_MS} AND NOT EXISTS (SELECT 1 FROM runner_offline_notices n WHERE n.runner_id = r.id AND n.notified_at >= r.last_seen_at)`, now, ) .toArray()[0]; return row?.due == null ? null : Number(row.due); } /** * Reports each of the account's runners once per outage, run by the account's alarm (runners live in their * owner's AccountState); a runner that checks in again clears its notice. */ export async function reportOfflineRunners(sql: Sql, now: number): Promise { sql.exec( `DELETE FROM runner_offline_notices WHERE runner_id IN ( SELECT n.runner_id FROM runner_offline_notices n LEFT JOIN runners r ON r.id = n.runner_id WHERE r.id IS NULL OR r.last_seen_at > n.notified_at)`, ); const offline = sql .exec( `SELECT r.id, r.name, r.repo_id, r.owner_id, r.last_seen_at FROM runners r WHERE r.last_seen_at IS NOT NULL AND r.last_seen_at < ? - ${OFFLINE_AFTER_MS} AND r.last_seen_at > ? - ${FORGET_AFTER_MS} AND NOT EXISTS (SELECT 1 FROM runner_offline_notices n WHERE n.runner_id = r.id) LIMIT ${RUNNERS_PER_SWEEP}`, now, now, ) .toArray(); for (const runner of offline) { sql.exec("INSERT OR IGNORE INTO runner_offline_notices (runner_id, notified_at) VALUES (?, ?)", runner.id, now); await notifyRunnerOffline(runner).catch((error: unknown) => console.error("Runner offline notice failed", error)); } }