/** * Graph Execution Engine v2 — Crash Recovery + Dispatch→Signal Reconcile * * Version: 3.0 * Date: 2026-07-25 * * The recovery half of the engine. When a process crashes mid-execution, the * persisted {@link EngineState} (written write-through by `engine-persistence.ts`) * survives, but the in-memory `onTaskTerminated` subscriptions and the advancing * critical section are gone. On restart the engine calls `recover()`, which: * * 1. Loads the persisted state (`engine-persistence.load`). * 2. Reconciles every `running` node against the live dispatch system * (`getTask(taskId)`): a vanished task → `timeout`; a task that finished * during the restart window → re-emit its terminating signal; a still-live * task → re-subscribe its `onTaskTerminated` listener. * 3. Rebuilds the frontier from every `ready` node. * 4. Drains the deferred completions (nodes that finished during the window). * * This is the exact pattern proven by `LoopCoordinator.reSubscribeListeners()` * (`src/loop/coordinator.ts:662-729`), which this module mirrors * (orphaned→interrupted, terminal→advance, live→re-subscribe). * * This module also owns the shared **status→signal mapping** used by BOTH the * live dispatch seam (`engine-advance.ts::_dispatchNode`) and recovery: * `completed → answer`, `error → escalate` (with `task.error`), `timeout → * escalate`, `cancelled →` no terminating signal (the node is cancelled * directly). Keeping the mapping here means the two entry points can never * drift apart. * * Finally, this module ships the stale-lock {@link EngineLockSweeper}, the * stale-node {@link NodeStalenessWatcher} (monitor M3), and the heartbeat-based * {@link NodeLivenessMonitor} (node-anomaly-detection subtask 3), matching * `src/loop/coordinator.ts:101-124` (engine-state-machine.md §3.4 / failure * resilience.md §5.6). All are **manually tickable** and do not start an * unbounded `setInterval` on their own — `start()` is opt-in so tests never * leak a timer. * * Hydrate/adopt completeness (H3 / L7 / M10): `hydrateEngineState` and * `adoptPriorNodeStates` carry the monitor-relevant graph state across * recovery and rebuilds — the append-only `checkpointHistory` (H3), per-node * `artifacts` / `evidence` and loop-group `convergenceFingerprint` (L7), and * the cross-restart `terminalNotified` dedup flags (M10). `subscribeTaskTermination` * returns its registered callback (M4) so callers can later * `removeTaskTerminatedListener`; `reconcileEngine` surfaces the re-subscribed * `{ taskId, callback }` pairs through an optional out-parameter. * * Design references: * - `.rolebox/design/engine-state-machine.md` §5.1 (recovery entry point), * §5.2 (idempotency), §5.4 (restart window), §5.6 (stale-lock sweeper). * - `.rolebox/design/failure-resilience.md` §5.1-§5.6. * - `src/loop/coordinator.ts:662-729` (proven re-subscribe pattern). */ import type { DispatchTask } from "../../dispatch/types.ts"; import type { UsageRecord } from "../../dispatch/budget/budget-tracker.ts"; import type { EngineState, NodeRuntimeState } from "../../types.engine-v2.ts"; import type { SignalType } from "./signal-bridge.ts"; import type { TaskTerminatedCallback } from "./dispatch-bridge.ts"; /** Error reason stamped on a node whose dispatch task vanished during restart. */ export declare const ORPHAN_REASON = "Worker task vanished during restart"; /** Dispatch statuses that ended the task — its termination was missed. */ export declare const TERMINAL_DISPATCH_STATUSES: Set; /** Dispatch statuses that are still live — re-subscribe, never re-dispatch. */ export declare const LIVE_DISPATCH_STATUSES: Set; /** * The dispatch surface recovery needs. Structurally satisfied by * {@link NodeDispatchPort} (`engine-advance.ts`) — a node dispatch port is a * superset of this. Both members are optional so test fakes and minimal ports * can omit the recovery half; every consumer guards on presence. */ export interface DispatchRecoveryPort { /** Look up a task's current status (mirrors `DispatchManager.getTask`). */ getTask?(taskId: string): DispatchTask | undefined; /** Register a one-time listener for a task's terminal transition. */ onTaskTerminated?(taskId: string, callback: TaskTerminatedCallback): void; /** * Cumulative token/cost usage for a single dispatched session (keyed by the * dispatch session ID). OPTIONAL-ADDITIVE — a port without it simply cannot * report per-node usage, so the engine degrades to the pre-Phase-7 behavior * (zero consumption). Structurally satisfied by {@link DispatchBridge} * (`dispatch-bridge.ts:getSessionUsage`). Consumed by {@link captureNodeUsage} * to populate `node.tokensConsumed` at task termination. */ getSessionUsage?(sessionId: string): UsageRecord; /** * List every task launched under a given parent session id. The graph id * seeds the dispatch parent session (`dispatch-bridge.ts:graphParentContext`), * so passing `state.graphId` returns every session this graph launched. * OPTIONAL-ADDITIVE — a port without it cannot sweep crash-window orphans (a * node persisted `running` with no `dispatchTaskId`), so the recovery sweep * degrades to record-nothing. Structurally satisfied by {@link DispatchBridge} * (`dispatch-bridge.ts:getTasksByParent`). */ getTasksByParent?(parentSessionId: string): DispatchTask[]; /** * Cancel a dispatched task. OPTIONAL-ADDITIVE — mirrors * `NodeDispatchPort.cancelTask` (`engine-advance.ts`); recovery uses it to * tear down the orphaned live session whose node recorded no `dispatchTaskId` * (it would otherwise keep burning budget with its result lost). Issued * fire-and-forget by the caller — never awaited. */ cancelTask?(taskId: string): Promise; } /** A dispatch status mapped to a terminating engine signal (answer | escalate). */ export interface DispatchStatusSignal { type: SignalType; payload: unknown; } /** A node whose completion was missed during restart, ready to re-emit. */ export interface DeferredSignal { nodeId: string; type: SignalType; payload: unknown; } /** What recovery reconciled, for diagnostics and tests. */ export interface ReconcileReport { /** Nodes whose live tasks were re-subscribed (waiting for termination). */ reSubscribed: string[]; /** Nodes whose tasks vanished → marked `timeout` (and re-emitted as escalate). */ timedOut: string[]; /** Nodes that finished during the window → their terminating signal re-emitted. */ deferred: DeferredSignal[]; /** * Crash-window orphan matches (Y2). A node is persisted `running` BEFORE * `_dispatchNode`'s `await executeNode(...)` resolves the task handle, so a * crash in that window leaves disk state `running` with NO `dispatchTaskId` — * yet the session was launched and is still live in the dispatch system. * Each entry pairs the orphaned node with the task id of the matched live * session (parent = graph id, description = `graph node ${nodeId}`); the * caller issues a fire-and-forget `cancelTask` so the orphan stops burning * budget. Record-only — reconcile never awaits and never cancels. Absent * when the port lacks `getTasksByParent` or no live task matched. */ orphanCancellations?: Array<{ nodeId: string; taskId: string; }>; } /** * Re-subscription handles surfaced to callers (monitor M4). Passed as an * optional out-parameter to {@link reconcileEngine}; when supplied, every live * re-subscription made by the pass is recorded here so the caller can later * remove the listener (via `removeTaskTerminatedListener(taskId, callback)`) * once the node is no longer running. */ export interface ReconcileSubscriptions { /** * Re-subscribed live tasks and the exact callbacks handed to * `port.onTaskTerminated`. The callback is the value returned by * {@link subscribeTaskTermination}, so it is safe to pass back to * `removeTaskTerminatedListener`. */ listeners: Array<{ taskId: string; callback: TaskTerminatedCallback; }>; } /** The signal-drive callback used to advance the graph on a task termination. */ export type RecoveryEmitSignal = (nodeId: string, type: SignalType, payload: unknown) => void; /** * Map a dispatch task's terminal status to the terminating engine signal that * advances its node: * * - `completed` → `answer` (forward data flow runs). When the subtask emitted a * real terminating signal (`task.terminatingSignal`), it is preserved * including the signal type (e.g. `revise_needed` / `escalate` / `answer`) * so loop back-edges and other signal consumers activate correctly. When no * terminating signal was recorded, the payload is tagged `{ __inferred: true }` * — this is a best-effort inference that the task "just finished", emitted so * downstream join evaluation and signal routing never silently stall. * - `error` → `escalate` (payload carries `task.error`). * - `timeout` → `escalate` (a timed-out worker is an unrecoverable failure). * - `cancelled` → `null` — a cancellation is not a terminating signal; the node * is cancelled directly by the caller instead. * - HITL statuses (`awaiting_approval` — the paused task's status — plus * `need_approval` / `blocked` / `need_clarification` — the signal types the * completion evaluator delivers) → the pausing `need_approval` signal. The * engine advances a declared `needs_approval` node `running → blocked` on it * (engine-advance.ts `_pauseForApproval`), so `[GRAPH BLOCKED]` fires instead * of the HITL termination being dropped at `subscribeTaskTermination`'s * `if (!sig) return;`. A node NOT declared `needs_approval` keeps today's * semantics (record-only; resumes when a human uses approve/reject). * - any other status → `null` (non-terminal: running / pending). */ export declare function mapDispatchStatusToSignal(status: string, task?: DispatchTask): DispatchStatusSignal | null; /** * Whether a dispatch task is currently STILL LIVE (running / pending / * awaiting_approval) per the dispatch port's authoritative `getTask` read. * * Backs the transient-error guard in {@link subscribeTaskTermination} and the * `_dispatchNode` race-guard (`engine-advance.ts`): when a dispatch termination * reports a status of `error`, the engine re-checks liveness before committing * the node to `escalate`. A transient execution error must never latch a node as * a terminal error while the underlying session continues — so if the * authoritative read shows the task is still live, the reported `error` is * treated as stale and the escalate is skipped (the node stays running). * * A missing `getTask` port, a vanished task, or a throwing read all resolve to * `false` (NOT live) — the conservative "genuine terminal" default — so a * reported error that cannot be re-confirmed as live still escalates. */ export declare function isDispatchTaskLive(port: DispatchRecoveryPort, taskId: string): boolean; /** * Record a node's per-session token/cost consumption into `node.tokensConsumed` * from the dispatch layer's budget tracker. * * Background: `node.tokensConsumed` was historically only ever assigned by * `adoptPriorNodeStates` (copying a prior run), so a freshly executed node * always reported zero per-node consumption. The dispatch subsystem tracks * usage per dispatched session (`BudgetTracker.getSessionUsage(sessionId)`, * keyed by the node's `dispatchSessionId`), so this helper reads that record at * task termination and writes it onto the node. * * Semantics (minimal honest path): * - **Replace, not accumulate.** The node's `tokensConsumed` is set to the * terminating session's usage. This is idempotent across the live-seam, * race-guard, and recovery paths (which may both observe the same * termination), so it cannot double-count. The residual gap — a node that * re-dispatches multiple sessions (retry / loop re-entry) reflects only the * LAST session's usage, not the cumulative total — is documented in * `docs/graph-engine-architecture.md`. * - **Zero-guard.** When the tracker reports all-zero usage (the session was * never sampled, or usage was reset), the node's existing value is left * untouched rather than clobbered to zero — so an adopted prior value is not * erased by a transient zero read. * - **Best-effort.** A throwing or absent `getSessionUsage` is a no-op that * never corrupts node advancement. * * @param state Engine state (used to mark the mutation dirty for persistence). * @param node The node whose `dispatchSessionId` identifies the session. * @param port The dispatch port exposing `getSessionUsage` (optional). */ export declare function captureNodeUsage(state: EngineState, node: NodeRuntimeState, port: DispatchRecoveryPort): void; /** * Register a one-time `onTaskTerminated` listener for a node's in-flight * dispatch task. This is the **single delivery seam** for dispatch→signal * routing — every dispatch termination (live or recovery) funnels through * {@link emitSignal}, which records a terminating signal and advances the * engine. The post-registration race-condition guard in _dispatchNode * (engine-advance.ts) catches already-terminal tasks via a deferred completion * instead of duplicating the emitSignal path, so this callback owns the * exclusive `signalBridge.record()` call. * * Guarded so a stale listener (from a superseded dispatch task id, or a node * the engine already moved past `running`) can never advance a node twice: * - the node's `dispatchTaskId` must still match the completed task id; * - the node must still be `running` (a cancellation that the engine already * applied — e.g. via the cascade canceller — or a deferred completion that * drained earlier in the same critical section — is skipped). * * When the dispatch task reports `cancelled`, the node is cancelled directly * (`running → cancelled`) — there is no terminating signal to record. * * Transient-error guard (subtask 4): a reported `error` is re-checked against * the dispatch port's authoritative `getTask` read via {@link isDispatchTaskLive}. * If the task is still live (running / pending / awaiting_approval), the `error` * is treated as stale — a transient execution error must not latch the node as a * terminal error while the underlying session continues — so the escalate is * skipped, the node stays `running`, and the listener is re-subscribed so a * genuine later termination still advances it. * * M9 listener-ledger escape (review 04-F5): the transient-error re-subscription * registers a NEW listener whose callback would otherwise escape the caller's * subscription ledger — dispose() could never unregister it and the zombie * callback would carry a disposed engine's `state`/`emitSignal` closures when it * finally fired. The optional `out` collector captures every such re-subscribed * `{ taskId, callback }` pair so the caller can register it into its ledger * (engine-advance.ts `_dispatchNode` passes its `_terminationSubscriptions`). * * Returns the exact callback handed to `port.onTaskTerminated` (monitor M4), so * a caller that no longer needs the subscription can pass it to * `removeTaskTerminatedListener(taskId, callback)`. Returns `undefined` when * there is nothing to subscribe (no task id, or a port without the listener * surface). */ export declare function subscribeTaskTermination(state: EngineState, port: DispatchRecoveryPort, node: NodeRuntimeState, emitSignal: RecoveryEmitSignal, out?: Array<{ taskId: string; callback: TaskTerminatedCallback; }>): TaskTerminatedCallback | undefined; /** * Reconcile every `running` node against the dispatch system * (failure-resilience.md §5.2): * * - no `dispatchTaskId`, or `getTask` returns undefined → orphaned → the node * is marked `timeout` with {@link ORPHAN_REASON} and queued for an `escalate` * re-emit so its failure propagates (it must not silently stall a join). * When the node was persisted `running` without a `dispatchTaskId` (the * crash-window orphan), the pass additionally sweeps the dispatch parent * session for the still-live task (`getTasksByParent` + the node-scoped * description) and records it in {@link ReconcileReport.orphanCancellations} * so the caller can cancel the orphaned session fire-and-forget. * - terminal dispatch status → the termination was missed during the window: * `cancelled` cancels the node in place; `completed` / `error` / `timeout` * queue the mapped terminating signal for re-emission. * - live status → re-subscribe the `onTaskTerminated` listener (the node keeps * running and will advance when the task finally terminates). * * This is a **pure decision pass** — it marks terminal nodes and records * re-subscriptions (and orphan-cancellation matches), but never dispatches and * never awaits. The caller re-emits {@link ReconcileReport.deferred} (awaiting * each), issues the recorded orphan cancellations fire-and-forget, and then * dispatches the rebuilt frontier. */ export declare function reconcileEngine(state: EngineState, port: DispatchRecoveryPort, emitSignal: RecoveryEmitSignal, out?: ReconcileSubscriptions): ReconcileReport; /** * Rebuild the frontier from every node whose lifecycle status is `ready`. * * Divergence from design: engine-state-machine.md §5.1 step 4 says "ready + * joinSatisfied". In the implemented engine a node is only transitioned to * `ready` when it is either a provisioned root (no join needed — * `joinSatisfied` stays false by construction) or a downstream node whose join * just became satisfied. So `status === ready` alone is the correct and complete * dispatch predicate — additionally requiring `joinSatisfied` would strand * ready root nodes. Every ready node is, by invariant, dispatchable. */ export declare function rebuildFrontier(state: EngineState): string[]; /** * Copy a loaded {@link EngineState} (from `engine-persistence.load`) into the * live state object the {@link AdvanceEngine} already references. Persistence * always saved this very object, so the loaded content is semantically * identical; copying in place keeps the advance engine's reference valid * (it reads `state.nodes` / `state.frontier` dynamically). */ export declare function hydrateEngineState(target: EngineState, source: EngineState): void; /** * Clear the durable `terminalNotified.blocked` claim for a graph whose hydrated * state is QUIESCENT-BLOCKED — no `running` / `ready` / `pending` node remains * and at least one node is `blocked`. * * Restart-refire correction (M10): the two-layer terminal dedupe (per-instance * {@link TerminationContext} + persisted `state.terminalNotified`) exists so a * terminal event fires exactly once across a rebuild/restart. But a * quiescent-blocked graph is a LIVE pause waiting on a human, not a completed * terminal — the crash that ended the previous process delivered its reminder, * while the restarting process has not. Because the restarted engine is freshly * constructed, its per-instance `ctx.terminalBlocked` is already `false` * (`engine-startup.ts` builds a new `EngineRuntimeImpl`); only the durable * layer survives and suppresses the re-fire. Clearing it here (leaving the * `complete` claim intact) lets the recover-side termination check re-fire the * `blocked` terminal exactly once. * * Strictly recover-scoped: called from {@link hydrateEngineState}, which is only * reached via `recover()` (plus tests), so the normal-session two-layer dedupe * semantics are never altered. * * @returns `true` when a quiescent-blocked `blocked` claim was cleared. */ export declare function clearRecoveredQuiescentBlockedGuard(state: EngineState): boolean; /** * Adopt a *prior* engine run's per-node progress into a freshly provisioned * state (same graph id, possibly a superset declaration). * * This backs the imperative `graph_*` tool flow where the toolset rebuilds a * fresh engine from the declaration after every mutation (`graph_add_node`) and * on every `graph_run`. Without adoption, a rebuild resets every node to * `ready`/`pending`, so a second `graph_run` re-dispatches nodes that already * completed — the "completed node re-run" bug. * * For every node present in BOTH states whose prior status is not `pending` * (i.e. it made real progress), the prior run's execution fields are copied * onto the freshly provisioned node **in place** (the target node object stays * bound to its owning state for checkpoint recording). The frontier is then * corrected: adopted non-`ready` nodes leave the frontier; adopted `ready` * nodes (re-entered by a prior retry/revise) are kept dispatchable. * * Nodes that exist only in the new declaration are untouched (fresh * `pending`/`ready` as provisioned). A prior node whose `agent` no longer * matches is skipped — its identity changed, so a fresh run is correct. * * Graph-level progress is carried too: budget counters, the signal ledger, * checkpoints, and loop-group traversal counts (so loop caps stay honest * across rebuilds). */ export declare function adoptPriorNodeStates(target: EngineState, prior: EngineState): void; /** * Clear the critical-section state a crashed process left behind: the * `advancingLock` may be stuck `true` (an exception escaped the `finally`, or * the process died mid-advance) and `pendingCompletions` may be non-empty. * In this fresh process no critical section is actually running, so the lock is * released and the orphaned deferred queue dropped (reconciliation re-derives * every running node's outcome from the dispatch system instead). */ export declare function clearStaleCriticalSection(state: EngineState): void; /** Options for {@link EngineLockSweeper}. */ export interface LockSweeperOptions { /** Sweep interval for `start()` (defaults to `SWEEPER_INTERVAL_MS`). */ intervalMs?: number; /** How long a lock may stay held before it is released (defaults to `ADVANCING_LOCK_TIMEOUT_MS`). */ lockTimeoutMs?: number; /** Called when the sweeper releases a stale lock. */ onRelease?: (graphId: string) => void; } /** * Stale-lock sweeper, mirroring `src/loop/coordinator.ts:101-124` * (engine-state-machine.md §3.4 / failure-resilience.md §5.6). The engine's * advancing critical section is guarded by `state.advancingLock`; if an * exception ever escapes the `finally` (or a process is killed mid-advance) the * lock can stay `true` and deadlock every subsequent signal. The sweeper * detects a lock held past {@link ADVANCING_LOCK_TIMEOUT_MS} and releases it. * * The engine tracks the lock with a plain boolean, so the sweeper remembers its * own first-observed-held timestamp per graph. A lock observed held for the * first time is recorded; it is only released once it has been *continuously* * held for the full timeout — a briefly-held lock is never touched. * * Test-control: the sweeper is **manually tickable** via {@link sweep} * (accepting an injectable `now` for deterministic tests) and never starts a * timer on its own. `start()` (the periodic `setInterval`) is opt-in — tests * drive {@link sweep} directly instead, so no unbounded interval leaks. */ export declare class EngineLockSweeper { private readonly opts; private readonly intervalMs; private readonly lockTimeoutMs; private timer?; private readonly firstSeen; constructor(opts?: LockSweeperOptions); /** * One sweep tick. Returns `true` when a stale lock was released. * * @param now Optional clock for deterministic tests (defaults to `Date.now`). */ sweep(state: EngineState, now?: number): boolean; /** Start the periodic sweep. Opt-in — never auto-started. */ start(state: EngineState): void; /** Stop the periodic sweep (no-op if never started). */ stop(): void; } /** Options for {@link NodeStalenessWatcher}. */ export interface NodeStalenessWatcherOptions { /** Tick interval for `start()` (defaults to `SWEEPER_INTERVAL_MS`). */ intervalMs?: number; /** * How long a `running` node may stay live before it is marked `timeout`. * A node with a declared per-node budget overrides this with * `node.budget?.timeout_ms` (the declaration is authoritative). */ nodeStaleTimeoutMs: number; /** * Called with the node id and error reason whenever a stale running node is * marked `timeout` by {@link tick}. */ onTimeout?: (nodeId: string, errorReason: string) => void; /** * Optional dispatch-liveness probe (the same quiet-but-alive channel the * {@link NodeLivenessMonitor} consults). When present and returning `true` * for a running node that is past its staleness deadline, {@link tick} * treats the node as quiet-but-alive rather than hung: it refreshes * `lastActivityAt` (heartbeatSource `"dispatch"`) and SKIPS the wall-clock * timeout for that tick. The node stays running while the dispatch layer * verifiably considers its task in-flight; the authoritative hung-kill for * a task that stays verifiably live but never completes lives in the * dispatch watchdog (`completion-evaluator.ts` not_ready branch), not here. * * When the probe is absent, returns `false` (task dead / orphaned), or * throws (unverifiable), the wall-clock deadline remains authoritative — * the node is marked `timeout` exactly as before (the "no-feed fallback" * contract). A throwing probe is swallowed and treated as not-alive, so a * tick never breaks and a broken probe can never resurrect a hung node. * When the probe is present and its node IS timed out, its result is * folded into the timeout REASON string for diagnostics (e.g. * `dispatch task live=false`). */ isDispatchAlive?: (node: NodeRuntimeState) => boolean; } /** * Stale-node watcher (monitor M3) — detects `running` nodes whose worker has * stopped advancing and marks them `timeout` so the graph never hangs on a node * nobody is driving. Same shape as the stale-lock {@link EngineLockSweeper}: * **manually tickable** (`tick` with an injectable clock for deterministic * tests) and never starts a timer on its own — `start()` (the periodic * `setInterval`) is opt-in, so tests never leak an interval. * * Deadline resolution per node: * - a node with a declared per-node budget (`node.budget?.timeout_ms`) uses * that value as its staleness deadline — the declaration wins; * - every other node uses the watcher-wide `nodeStaleTimeoutMs`; * - a per-node `0` disables staleness for THAT node (the documented per-node * opt-out sentinel, pinned by engine-recovery.test.ts); a non-positive * watcher-wide `nodeStaleTimeoutMs` disables staleness for every node; * - a NEGATIVE per-node override is invalid input (rejected by zod + * validator-v2 rule 10); defensively it is NOT treated as an opt-out — * it falls back to the watcher-wide default so a malformed override can * never silently disable the watchdog for a node that did not opt out. * * Dispatch-liveness gate (S2): when the optional {@link NodeStalenessWatcherOptions * .isDispatchAlive} probe is present and verifiably reports the node's task * in-flight, a running node past its deadline is quiet-but-alive — the tick * refreshes its heartbeat (heartbeatSource `"dispatch"`) and skips the * timeout. The wall-clock kill remains the no-feed fallback: probe absent, * `false`, or throwing → the node is timed out byte-identically to the * legacy behavior, and the authoritative hung-kill for a verifiably-live * but never-completing task lives in the dispatch watchdog * (`completion-evaluator.ts` not_ready branch). * * The engine's behavior is unchanged unless a consumer instantiates the * watcher (default not instantiated) and drives it — `index.ts` (S7) wires * the opt-in interval. Each tick marks stale nodes via * {@link markTimedOut} (a normal `running → timeout` transition, so lifecycle * checkpoints and the critical-dirty flag are recorded by the shared * transition choke point). */ export declare class NodeStalenessWatcher { private readonly opts; private readonly intervalMs; private readonly nodeStaleTimeoutMs; private timer?; /** * Consecutive failed periodic ticks (Y22). Non-zero means the watchdog is * DEGRADED — a tick threw and therefore never ran. Reset by the next * successful tick; the count throttles the degradation log line. */ private tickFailures; constructor(opts: NodeStalenessWatcherOptions); /** Consecutive failed periodic ticks since the last successful one (Y22). */ get consecutiveTickFailures(): number; /** Whether the periodic tick is currently degraded (≥1 consecutive failure). */ get tickDegraded(): boolean; /** * One staleness tick. Marks every `running` node whose elapsed `startedAt` * time meets or exceeds its staleness deadline as `timeout` (via * {@link markTimedOut}), reporting each through the `onTimeout` callback — * unless the optional dispatch-liveness probe verifiably reports the node's * task in-flight, in which case the node is quiet-but-alive: its heartbeat * is refreshed (heartbeatSource `"dispatch"`) and the timeout is skipped * for this tick. * * @returns The ids of the nodes that were timed out by this tick. * @param now Optional clock for deterministic tests (defaults to `Date.now`). */ tick(state: EngineState, now?: number): string[]; /** * Evaluate the optional dispatch-liveness probe for a node — exactly once * per tick. Returns `undefined` when the probe is absent or throws; both * resolve to the conservative not-verifiable default, so a broken probe can * never resurrect a hung node and a tick never breaks. */ private evalDispatchAlive; /** * Compose the timeout reason for a stale running node. The base message is * unchanged; the node's liveness-carrier facts (idle time since the last * heartbeat, heartbeat source, stall classification) and — when the * optional dispatch-liveness probe is present — its result are appended for * diagnosis (S1 enrichment). The probe result is passed in precomputed by * {@link tick} (evaluated exactly once per node per tick — the same value * that gated the timeout skip), so the reason is always consistent with the * decision. Reason string only: this NEVER influences which nodes time out * (the {@link tick} gate plus the wall-clock deadline decide); a throwing * probe resolves to `undefined` (segment omitted) so a tick never breaks; * the legacy reason is byte-identical when no liveness facts or probe * exist. */ private buildTimeoutReason; /** Start the periodic tick. Opt-in — never auto-started. */ start(state: EngineState): void; /** Stop the periodic tick (no-op if never started). */ stop(): void; } /** * The immutable detection facts captured at the moment {@link NodeLivenessMonitor} * fires its `onStall` callback. Passed as the optional third argument so a * consumer (e.g. the engine's stall notifier seam) can render an actionable * reminder without re-deriving monitor internals. */ export interface StallDetectionInfo { /** Idle time (ms) since the node's last heartbeat at detection time. */ idleMs: number; /** The soft-stall warn threshold (ms) used for this detection. */ stallWarnMs: number; /** Epoch-ms timestamp stamping the start of this stall episode. */ stallWarnedAt: number; } /** * A node-stall event emitted via the optional {@link NodeLivenessMonitorOptions * .onStall} callback seam (node-anomaly-detection subtask 5). The engine * packages only the immutable facts captured at detection time; notification / * delivery (a notifier) is the consumer's concern and never lives in the * monitor or the engine. */ export interface NodeStallEvent { /** Owning graph id. */ graphId: string; /** The node that entered the soft-stall (`stalling`) classification. */ nodeId: string; /** The node's bound agent id. */ agent: string; /** Idle time (ms) since the node's last heartbeat at detection time. */ idleMs: number; /** The soft-stall warn threshold (ms) used for this detection. */ stallWarnMs: number; /** * Epoch-ms timestamp when this stall episode was first warned. Identifies * the stall episode — a notifier's dedupe key folds it in so a recovery * (fresh heartbeat → `healthy`) followed by a re-stall is a distinct episode * and legally re-notifies, while an idempotent replay of the same episode is * dropped. */ stallWarnedAt: number; } /** Options for {@link NodeLivenessMonitor}. */ export interface NodeLivenessMonitorOptions { /** Tick interval for `start()` (defaults to `SWEEPER_INTERVAL_MS`). */ intervalMs?: number; /** * Watcher-wide staleness timeout — the hard cap on how long any `running` * node may stay alive. The per-node effective deadline is * `min(node.budget?.timeout_ms ?? this, this)` — a node's declared budget * can shorten the window but never extend it past this value. A * non-positive effective deadline disables liveness-based staleness for * that node. */ nodeStaleTimeoutMs: number; /** * Idle time since the last heartbeat at which a node is first classified * `stalling` and `onStall` fires. Single-fire per stall episode — the * callback does not repeat while the node stays `stalling`, and a fresh * episode (after a heartbeat returns the node to `healthy`) warns again. * Defaults to `min(60_000, nodeStaleTimeoutMs / 2)`. */ stallWarnMs?: number; /** * Additional idle time past `stallWarnMs` before a stalling node is * hard-stalled — marked `timeout` via {@link markTimedOut} and reported * through `onTimeout` (the same signature as * {@link NodeStalenessWatcherOptions.onTimeout}). Defaults to 30_000. */ stallGraceMs?: number; /** * Optional dispatch-liveness probe (quiet-but-alive channel). When present * and returning `true` for a running heartbeat-fed node, {@link tick} * treats the node as quiet-but-alive rather than stalled: once the node has * been idle past `stallWarnMs` (i.e. it is ABOUT to be classified stalled), * the tick refreshes `lastActivityAt` (heartbeatSource `"dispatch"`) and * skips the stall ladder. * * The probe must answer "is the underlying dispatch/process verifiably * in-flight?" — on opencode, the background task status being * running/pending/awaiting_approval (backed by the SDK session tracking); * on Pi, the task status running (backed by the live child process between * JSON events). A subagent mid-turn with zero streaming events is alive, not * stalled, so it must not be hard-stalled merely for being quiet. * * Consequences (deliberate): * - The warn ladder (`[GRAPH NODE STALLED]`) and the heartbeat hard-stall * now fire ONLY when the dispatch can no longer verify the task — the * genuinely abnormal state (orphaned node, dead task that never * terminal-advanced). Quiet-but-alive nodes no longer warn. * - A genuinely HUNG node whose task stays verifiably live but never * completes is NOT caught here — it falls to the dispatch watchdog's * not_ready hung-kill (`completion-evaluator.ts`), and the wall-clock * {@link NodeStalenessWatcher} wired beside this monitor applies the SAME * probe gate (S2): while the probe verifies the task in-flight, the * watcher skips its timeout too. Only a watcher WITHOUT the probe * (feed-less engines / test fakes) keeps the pure wall-clock deadline as * the hung-but-alive backstop. * - A node whose declared per-node budget (`budget.timeout_ms`) is tighter * than the warn window is bounded by that cap (the hard-stall branch * precedes the probe refresh) — the declared deadline is authoritative. */ isDispatchAlive?: (node: NodeRuntimeState) => boolean; /** * Called once when a running node first enters the soft-stall * (`stalling`) classification. The optional third argument carries the * {@link StallDetectionInfo} captured at fire time (idle / warn threshold / * episode timestamp) so a consumer can render an actionable notification * without re-deriving monitor internals. The monitor contains the call in * try/catch — a throwing consumer is logged and swallowed so a tick never * breaks. */ onStall?: (nodeId: string, reason: string, info?: StallDetectionInfo) => void; /** * Called with the node id and error reason whenever a hard-stalled running * node is marked `timeout` by {@link tick}. */ onTimeout?: (nodeId: string, errorReason: string) => void; } /** * Node liveness monitor — heartbeat-based stall detection layered on top of * the wall-clock {@link NodeStalenessWatcher} (node-anomaly-detection subtask * 3). Same shape as the watcher: **manually tickable** (`tick` with an * injectable clock for deterministic tests) and never starts a timer on its * own — `start()` (the periodic `setInterval`) is opt-in, so tests never leak * an interval. * * Unlike the wall-clock watcher (which times a node out purely from * `startedAt`), the monitor classifies a node from its **heartbeat feed** * (`node.liveness.lastActivityAt`, written by subtask 2's * `recordLivenessHeartbeat` / the platform liveness feed): * * - heartbeat fresh (`now - lastActivityAt < stallWarnMs`) → `healthy` — the * node's `stallStatus` is reset to `healthy`, clearing any soft-stall * classification; * - soft stall (Tier 1) — idle `>= stallWarnMs` and `< stallWarnMs + * stallGraceMs` → the node is classified `stalling` with `stallWarnedAt` * stamped, and `onStall` fires **once** (guarded on the existing * `stallStatus`, so the warning never repeats within one episode); * - hard stall (Tier 2) — idle `>= min(effectiveDeadline, stallWarnMs + * stallGraceMs)` → the node is marked `timeout` via the shared * {@link markTimedOut} (the normal `running → timeout` transition, so * lifecycle checkpoints and the critical-dirty flag are recorded by the * transition choke point) and reported through `onTimeout`. * * Fallback (Tier 3): a node WITHOUT a heartbeat feed (`liveness.lastActivityAt` * absent) is skipped entirely — it keeps the pure wall-clock deadline of the * unmodified {@link NodeStalenessWatcher}. Both monitors coexist; the engine's * behavior is unchanged unless a consumer instantiates the monitor (default * not instantiated) and drives it. */ export declare class NodeLivenessMonitor { private readonly opts; private readonly intervalMs; private readonly nodeStaleTimeoutMs; private readonly stallWarnMs; private readonly stallGraceMs; private timer?; /** * Consecutive failed periodic ticks (Y22). Non-zero means the monitor is * DEGRADED — a tick threw and therefore never ran. Reset by the next * successful tick; the count throttles the degradation log line. */ private tickFailures; constructor(opts: NodeLivenessMonitorOptions); /** * One liveness tick. Classifies every `running` node with a heartbeat feed * (see the class docs for the healthy → stalling → stalled ladder) and * hard-stalls nodes past their effective deadline via {@link markTimedOut}. * * @returns The ids of the nodes that were hard-stalled (timed out) by this tick. * @param now Optional clock for deterministic tests (defaults to `Date.now`). */ tick(state: EngineState, now?: number): string[]; /** Consecutive failed periodic ticks since the last successful one (Y22). */ get consecutiveTickFailures(): number; /** Whether the periodic tick is currently degraded (≥1 consecutive failure). */ get tickDegraded(): boolean; /** Start the periodic tick. Opt-in — never auto-started. */ start(state: EngineState): void; /** Stop the periodic tick (no-op if never started). */ stop(): void; } //# sourceMappingURL=engine-recovery.d.ts.map