import type { CoreEvent, WorkerStallEscalatedFrame, WorkerStalledFrame } from '../core/types.js'; export type { WorkerStalledFrame, WorkerStallEscalatedFrame }; /** Default sweep cadence — frequent enough that `quietForMs` is at most ~30s stale. */ export declare const DEFAULT_SWEEP_INTERVAL_MS = 30000; /** * Bound on a single sweep's engine round-trip (`listExecuting()`, `reassign()`), comfortably * under the sweep cadence (crew#442): the re-entrancy guard's `finally` only releases once its * awaited call SETTLES, so an engine call that never settles — plausible while a run is * genuinely wedged, or during a governed-run council's capacity spike — used to pin the guard * forever, silently killing detection AND escalation for every run the daemon watches. A bounded * timeout guarantees the guard always releases, degrading a hang to one skipped/failed attempt. */ export declare const SWEEP_ENGINE_TIMEOUT_MS = 10000; /** A run is sampled for process liveness once it has been quiet this long (crew#629). */ export declare const LIVENESS_SAMPLE_AFTER_MS = 60000; /** The slice of a run the sweep needs; the server maps it from `sessionsDetail()`. */ export interface ExecutingRun { id: string; /** The run's CURSOR unit ord when the caller knows it — what `reassignUnit` must name. */ ord?: number; /** The cursor unit's assigned seat — the seat a reassign moves AWAY from. Absent = unknown * (a reassign then lets the engine's council pick). */ cli?: string; /** The run's own seat pool (`session.clis`), in roster order — the failover candidates a * reassign may route to (perf#4). Absent/empty = no pool known: reassign in place. */ seats?: string[]; /** Seats a failover must NOT land on, so evaluator ≠ creator holds across a stall reassign * (DES-L3 PR-3E; crew#638, crew#583): an EVALUATOR cursor avoids the seats that built the * work it reviews, a CREATOR cursor avoids the run's evaluator seats, and a unit the engine * routed `evaluator_distinct` avoids the seat it was routed away from. Absent/empty = no * constraint (a free-text unit carries no role). */ avoid?: string[]; /** Seats THIS RUN benched (`session.benched_seats`, crew#833): a seat the engine already failed * over away from — quota exhausted, signed out, dead — is never a failover target in the same * run, whatever the evaluator ≠ creator read says. With the reason, so the needs-you frame's log * can say why. Absent/empty = nothing benched (or an older engine). */ benched?: Array<{ cli: string; reason: string; }>; /** What runs the cursor unit (crew #580 / #581): a `tool` unit is the engine's own command — * there is no seat to fail over to, so the escalation stage NOTIFIES instead of reassigning. * Absent = unknown (older engine views): today's ladder. */ executor?: 'tool' | 'agent'; } /** * The escalation stage's per-sweep configuration (crew#341), resolved live so a PUT /settings * change applies without a restart. `minutes` absent/0/invalid = escalation OFF (the default). */ export interface StallEscalationConfig { /** Quiet minutes before the watchdog ACTS. Clamped to at least the detection threshold. */ minutes?: number | undefined; /** What an armed escalation does (default `reassign`). */ action?: 'reassign' | 'notify' | undefined; /** Automatic reassigns per run before fail-loud (default 2; clamped to ≥ 1). */ maxPerRun?: number | undefined; /** * The SILENT-TURN rung (crew#972): quiet minutes after which an armed watchdog acts early on a * cursor unit whose seat has streamed NO output since its dispatch. Absent/0/invalid = off (the * daemon wires the `workerSilentTurnMinutes` setting, default 8). Never applies to a tool unit, to * a worker that returned, or while the run's processes or seat containers are busy. */ silentTurnMinutes?: number | undefined; } export interface StallWatchdogDeps { /** Runs whose engine status is `executing`, right now. Read per sweep, never cached. */ listExecuting: () => Promise; /** The /ws fan-out for the synthetic frames (`events/bus.ts` `broadcast` in the daemon). */ broadcast: (frame: WorkerStalledFrame | WorkerStallEscalatedFrame) => void; /** Threshold resolver, minutes — read per sweep so a PUT /settings change applies live. */ stallMinutes?: () => number | Promise | undefined; /** * The escalation stage (crew#341). Absent = the watchdog is detection-only (crew#287), * regardless of settings — a server that does not wire recovery cannot perform it. */ escalation?: { /** Resolved per sweep; `minutes` absent/0/invalid keeps escalation OFF. */ config: () => StallEscalationConfig | undefined | Promise; /** * Recycle the wedged cursor unit (the daemon wires `adapter.reassignUnit`). `cli` names * the seat to re-dispatch to — the watchdog passes its failover TARGET (a different seat * from the run's pool when one exists, else the current seat recycled in place); absent = * the engine re-runs the council. */ reassign: (runId: string, ord: number, cli?: string) => Promise; }; /** * Every frame this watchdog broadcasts, handed over for RECORDING (wicked-studio#284): the daemon * writes one audit line per frame (`run.stall.detected` / `run.stall.escalated`) and fills the * index `GET /runs/:id/events` reads. Absent = frames are live-only (`/ws`), the pre-#284 * behaviour — a directly-driven watchdog in a test records nothing. */ onFrame?: (frame: WorkerStalledFrame | WorkerStallEscalatedFrame) => void; /** * Called when a relayed `unitOutputCaptured` carries `stepStatus: "timed_out"` — the ENGINE's * own turn ceiling fired (the run is terminating; the last-resort backstop the silence ladder * exists to preempt). The server wires an audit entry + warn log so the timeout is * operator-visible as WHAT IT IS rather than as an anonymous cancel. Versioned safely: an * older engine never sends the value, so this never fires — and `"cancelled"` (ambiguous on * old engines: operator OR timeout) deliberately triggers NOTHING, because acting on an * operator's cancel is the failure mode this whole seam exists to prevent. */ onTurnTimeout?: (info: { session: string; ord?: number; attempt?: number; }) => void; /** * Process liveness (crew#629): for the given quiet runs, the ones whose own processes (the * worker, a test runner or build it started, the checks floor) burned CPU since the last sweep, * mapped to a short description of the busiest process. A busy run is not silent: its clock * re-arms exactly as an event would. The daemon wires `RunLivenessSampler`; absent = the * frame-only clock. */ busy?: (runIds: string[]) => Promise>; /** Clock (tests stub it). */ now?: () => number; /** Wired to the daemon's warn logger — a stall is an operator-attention signal. */ log?: (m: string) => void; } export declare class WorkerStallWatchdog { private readonly deps; /** run id → epoch ms of the last engine event observed for it (or the seed, see sweep). */ private readonly lastEventAt; /** DES-SYSTEM-VIEW: the newest EVIDENCE of life per run — a relay frame, or a busy child process. * Kept apart from `lastEventAt`, which a first observation also seeds with "now" (a clock start, * not evidence), so the system snapshot never reads a seeded clock as a sign of work. */ private readonly evidenceAt; /** run id → last unit ord any of its events named. */ private readonly lastOrd; /** Per-run notification/action latches for the CURRENT quiet period (cleared by any event). */ private readonly quietPeriods; /** run id → automatic reassigns consumed (the crew#341 budget). Pruned with the run. */ private readonly escalationCount; /** * run id → seats already stall-reassigned AWAY from on this run, so consecutive escalations * rotate through the pool instead of bouncing back to a seat that already wedged (perf#4). * WATCHDOG-LOCAL BY DESIGN: a stalled seat is not an errored seat — this memory must never * reach the engine's `worker_failed_clis` (resume-path exclusion) or the SeatHealthTracker. * Pruned with the run, like the budget. */ private readonly stalledSeats; /** * run id → the ord whose worker RETURNED (`unitOutputCaptured`) and whose gate evaluation is in * flight (crew#581). Cleared by the next dispatch. A reassign here would start a second creator * over finished work, so the watchdog notifies instead. */ private readonly evaluating; /** run id → `unitOutputDelta` frames since the last dispatch: ≤ 1 is a startup banner, no work * (crew#638), which is how an exhausted budget names a seat that produced nothing. */ private readonly outputSinceDispatch; private readonly now; private readonly log; private handle; private sweeping; constructor(deps: StallWatchdogDeps); /** Fold one relay frame: any event for a run is proof of life — stamp its clock and RE-ARM. */ ingest(event: CoreEvent): void; /** The newest evidence of life this process saw for `runId` (a relay frame or a busy child), or * `undefined` when it saw none — never the seeded clock of a first observation. */ evidence(runId: string): { at: number; source: 'relay' | 'process'; } | undefined; /** One detection pass. Public so tests drive it directly; the armed interval calls it too. */ sweep(): Promise; /** Arm the periodic sweep. Unref'd — the daemon's server lifecycle owns shutdown. */ start(intervalMs?: number): void; stop(): void; /** * The runs quiet for at least a minute whose processes are busy (crew#629). Sampled every sweep * while any run is quiet, so the CPU comparison spans one sweep interval. A failing or slow * probe reads as "no run busy": the frame-only clock, never a false proof of life. */ private busyRuns; /** Threshold in ms — an unset/invalid setting falls back to the 15-minute default. */ private thresholdMs; /** * Resolve the escalation stage for THIS sweep, or `undefined` while it is off. Fail-safe on * a throwing config read: acting on a run is exactly the wrong response to uncertainty, so a * failed read logs and keeps this sweep detection-only. */ private escalationArmed; /** * (crew#972) Whether `run`'s cursor seat has streamed NOTHING since its dispatch: an agent unit * (never a tool command), dispatched while this process watched (a count of 0 — unknown after a * restart reads as not silent), whose worker has not returned (no gate evaluation in flight). */ private silentTurn; /** Perform one escalation for `run`, emit its `workerStallEscalated` frame, audit, log. */ private escalate; /** * Put a frame on `/ws` and hand it to the recorder — the ONE place a watchdog frame leaves this * class. The recorder (the daemon's `onFrame`) writes the durable audit line and fills the index * `GET /runs/:id/events` serves; a throwing recorder is logged and swallowed, because recording * a stall must never break detecting the next one. */ private emit; /** * The failover TARGET for a stall reassign (perf#4): the first seat of the run's own pool * that is neither the currently-stalled seat, nor one this run already stall-reassigned away * from, nor one this run BENCHED (crew#833 — `benched` lists the pool seats the bench excluded). `undefined` = no distinct candidate (no pool known, a single-seat pool, or every * other seat already stalled here) — the caller then recycles in place, which is today's * (still safe) behaviour. Pool order is `session.clis` order: deterministic, no health * heuristics — the per-run budget bounds how far the rotation can walk. * * `avoided` lists the candidates `run.avoid` excluded (evaluator ≠ creator). When it is * non-empty and no target is left, the caller escalates to a human instead of recycling in * place (crew#638). */ private pickFailoverSeat; /** Returns the independent detection/action latches for this quiet period. */ private quietPeriod; } //# sourceMappingURL=stall-watchdog.d.ts.map