import type { FleetConfig } from '../config.js'; import type { ResolvedWatchdog } from './config.js'; import { executeNotifierRun, executeWatchdogRun } from './run.js'; export interface WatchdogSchedulerState { version: 1; consecutiveFailures: number; heldDown: boolean; heldSince?: string; lastRunAt?: string; nextRunAt?: string; lastError?: string; } /** Read a watchdog's scheduler state; a missing or corrupt file starts clean (mirrors readRestartLedger). */ export declare function readSchedulerState(name: string): WatchdogSchedulerState; /** Write never throws: scheduler diagnostics must never take the loop down. */ export declare function writeSchedulerState(name: string, s: WatchdogSchedulerState): void; /** * Operator release clears failures and heldDown so a held-down * loop's next held-down poll sees a clean state and resumes running. Also * clears the ledger's `heldDownAlerted` flag — that flag is what makes "alert * once per hold-down" durable across scheduler restarts (it lives in * alerts.json, not state.json), so a release must reset it too or a * subsequent hold-down would silently alert zero times. * * Also reclaims a demonstrably stale run lock: * a watchdog SIGKILLed mid-run (e.g. systemd's TimeoutStopSec on a * fleet-wide stop) leaves `.run-lock` behind forever — the loop's `finally` * that would normally release it never runs. Without this, every future tick * sees the lock held and reports `skipped_overlap` indefinitely, and skips * never alert. `ours-fleet restart ` is the documented recovery, so * it must clear a stale lock too, not just the failure/hold-down bookkeeping. * But only a DEMONSTRABLY stale lock (dead owner pid, or legacy lock with no * owner metadata) — a lock genuinely held by a live run (foreground * `watchdog-run`, another scheduler instance) must survive an operator's * `restart` of a DIFFERENT problem (e.g. releasing hold-down) unrelated to * that live run; two runs sharing the same temp dir would corrupt each * other's output. Best-effort: a reclaim failure here must not turn an * operator's recovery action into a crash. */ export declare function resetSchedulerState(name: string): void; export declare const WATCHDOG_HOLD_THRESHOLD = 3; export declare const WATCHDOG_BACKOFF_MAX_MS = 3600000; /** Bounded exponential backoff: 1x, 2x, 4x, ... capped at WATCHDOG_BACKOFF_MAX_MS. */ export declare function watchdogBackoffMs(intervalMs: number, failures: number): number; export interface SchedulerDeps { now(): Date; sleep(ms: number): Promise; log(line: string): void; binPath: string; /** * The fleet config `runScheduler` loaded from the `-c FILE`/default path * Threaded into both `runOnceFor`'s and the notifier's * deps so a run under a non-default config doesn't silently fall back to * `loadConfig()`'s default `~/fleet.yaml`. `runWatchdogLoop` callers that * bypass `runScheduler` (tests) may omit it — the run/notifier machinery * falls back to `loadConfig()` itself when `cfg` is undefined. */ cfg?: FleetConfig; /** Injectable for tests. */ runOnceFor?: typeof executeWatchdogRun; /** Loop exit for tests + SIGTERM (wired by the CLI, not here). */ shouldStop?(): boolean; /** * Scheduler-level alert hook: fired once per hold-down * transition (guarded in `settle` by `ledger.heldDownAlerted`, cleared by * `resetSchedulerState`). Default: `executeNotifierRun` — the fleet * process can't message on its own (deviation 4), so the default delivers * the alert via a one-shot notifier agent under the watchdog's identity. */ onSchedulerAlert?(wd: ResolvedWatchdog, text: string): Promise; /** * Injectable notifier launcher backing the default `onSchedulerAlert` — * consulted only when `onSchedulerAlert` itself is not supplied. Default: * `executeNotifierRun`. Lets tests observe/short-circuit the default alert * path (binPath/log/now/sleep wiring) without replacing onSchedulerAlert * wholesale. */ notifierRun?: typeof executeNotifierRun; /** Poll cadence while held down. Default 5000. */ heldPollMs?: number; /** * Injectable run-lock primitives. Default: store's acquireRunLock / * releaseRunLock. Both are mkdir/rmdir-backed and can throw (EACCES, EIO, * ENOTEMPTY on release) — the loop always classifies such a throw as a * failed tick rather than letting it escape. */ locks?: { acquire(name: string): boolean; release(name: string): void; }; } /** * One watchdog's scheduling loop: run immediately, then repeatedly sleep the * backed-off interval and run again, until `shouldStop()`. No overlap (a run * lock guards each attempt), bounded exponential backoff on failure, and a * hold-down circuit breaker after WATCHDOG_HOLD_THRESHOLD consecutive * failures (released externally via resetSchedulerState). */ export declare function runWatchdogLoop(wd: ResolvedWatchdog, deps: SchedulerDeps): Promise; /** * Run every enabled watchdog's loop concurrently until deps.shouldStop(). * SIGTERM wiring into shouldStop is the CLI's job, not this * function's. */ export declare function runScheduler(configPath: string | undefined, deps: SchedulerDeps): Promise;