/** * Runner failure episodes. * * Measured 2026-08-20/21: the control plane was broken for 16 hours and the * runner knew — 822 consecutive claim failures — but all it could do was * write one opaque journal line per poll, and a oneshot timer lost even the * in-memory streak at every process exit. Nobody saw anything until a human * looked. This module gives the runner ONE persisted failure episode per * outage: state on disk (surviving process exits), ONE structured event when * the episode opens, silent count updates while it stays open, and ONE * recovery event when the plane comes back. * * OPACITY (binding): episode state and events carry NO error text — only a * normalized failure class derived from safe signals (instanceof against this * package's own error types, numeric HTTP status), and a sanitized runnerId. * Foreign error messages are exactly where credentials have been observed to * live, so they are never interpolated, logged, or persisted here. * * DELIVERY BOUNDARY (binding): this package owns detection plus a generic * outbox/notifier hook. There is NO hardcoded destination (no #incidents, no * conversations dependency): fleet deployment binds * `LOOPS_RUNNER_NOTIFIER_CMD` and reads the outbox file. The runner never * blocks on escalation: notifier and state I/O failures are swallowed and * retried on the next poll, and lock contention SKIPS an update rather than * waiting — a poll is never delayed by episode tracking. * * EXACTLY-ONCE EVENTS: the episodeId is deterministic (derived from the * streak's persisted firstFailureAt and the runnerId), and each update runs * inside a short-lived lock file. Every crash window converges to one open * event and one recovery event per episode. * * DELIVERY-CONFIRMATION STATE (findings F3/F4 of the PR #778 successor): * dedup lives in the STATE FILE, never in a scan of the outbox history. Each * append attempt records, in the same state write that marks the event * pending, the outbox byte offset at which the attempt begins * (`streak.pendingAppend.fromOffset`). A retry then scans EXACTLY the bytes * appended since that offset — by construction the only region that can hold * the attempted line (appends are O_APPEND single writes, and the offset is * taken before the append). The scan is bounded by the inter-attempt delta * (normally zero), never by the outbox's size or history, and it cannot miss: * if the earlier append landed, its line is inside the window. A fresh emit * (no pending record) appends directly without any scan. The notifier fires * ONLY when the append is freshly made — a retry that finds the event already * present (lost confirm-write) deduplicates and does not re-notify. * * CONTENTION-PERSISTED SUCCESSES (finding F2): a success observed while the * lock is contended used to be dropped — the recovery transition was lost if * those were the final two successes before the runner exited. Contended * successes now append one line to `.success-intent.jsonl` * (no lock needed — single atomic O_APPEND writes), and the next lock holder * drains them under the lock BEFORE applying its own observation, so the * recovery transition is never lost, only deferred by the contention window. * * OWNERSHIP-CHECKED LOCK RELEASE (finding F5): each lock file carries a * per-acquisition token. A holder deletes the lock ONLY if it still owns it * (the file content is still its token). A displaced live holder — whose lock * was taken over after a 10s-stale takeover — therefore can never delete the * successor's lock, so a critical section cannot open a concurrent entry. * * The critical section is µs-scale (bounded state read, bounded append * window, one rename), so acquisition is a short bounded retry: contention * means another runner process is mid-update, and after the retries the * update is SKIPPED (or, for successes, persisted as an intent) — never an * exception, never an unbounded wait. */ /** Episode opens after this many consecutive claim/poll failures… */ export declare const EPISODE_OPEN_CONSECUTIVE_FAILURES = 3; /** …whose streak spans at least this long (a 3-flicker blip is not an outage). */ export declare const EPISODE_OPEN_SPAN_MS = 120000; /** Episode closes after this many consecutive successful polls. */ export declare const EPISODE_CLOSE_SUCCESSES = 2; /** Environment variable holding the optional notifier command (fleet binding surface). */ export declare const LOOPS_RUNNER_NOTIFIER_CMD_ENV = "LOOPS_RUNNER_NOTIFIER_CMD"; /** A notifier that has not exited by this point is killed; escalation must not accumulate orphans. */ export declare const NOTIFIER_KILL_MS = 30000; export type RunnerFailureClass = "connectivity" | "http_5xx" | "auth" | "contract" | "refusal"; export declare function classifyRunnerFailure(error: unknown): RunnerFailureClass; export type RunnerEpisodeDeliveryState = "counting" | "open" | "open_pending" | "recovery_pending"; export interface RunnerEpisodeStreak { firstFailureAt: string; lastFailureAt: string; consecutiveCount: number; failureClass: RunnerFailureClass; /** Present once the streak has become an episode. */ episodeId?: string; openedAt?: string; /** Consecutive successful polls since the episode opened (close at 2). */ consecutiveSuccesses?: number; deliveryState: RunnerEpisodeDeliveryState; /** * Delivery-confirmation state (findings F3/F4): present while the event * this episode is currently trying to deliver (the open event while * `deliveryState === "open_pending"`, the recovery event while * `"recovery_pending"`) has had an append attempt whose confirm-write is * not yet durable. `fromOffset` is the outbox byte size when the attempt * began; a retry scans exactly `[fromOffset..EOF]` — the only region that * can hold the attempted line. Absent means no append has been attempted * (fresh emits append directly, no scan). */ pendingAppend?: { fromOffset: number; attemptedAt: string; }; } /** Shape of `/runner-episodes.json`. `streak` doubles as the pre-open failure counter. */ export interface RunnerEpisodesFile { version: 1; runnerId?: string; streak?: RunnerEpisodeStreak; lastSuccessAt?: string; } export type RunnerEpisodeEvent = { evt: "loops_runner_control_plane_unreachable"; episodeId: string; runnerId: string; firstFailureAt: string; lastFailureAt: string; openedAt: string; consecutiveCount: number; failureClass: RunnerFailureClass; lastSuccessAt: string | null; } | { evt: "loops_runner_control_plane_recovered"; episodeId: string; runnerId: string; firstFailureAt: string; openedAt: string; recoveredAt: string; consecutiveCount: number; failureClass: RunnerFailureClass; outageMs: number; }; export interface RunnerEpisodeRecorder { /** Record one failed claim/poll. Never throws. */ recordFailure(error: unknown): void; /** Record one successful poll. Never throws. */ recordSuccess(): void; } export interface RunnerEpisodeRecorderOptions { /** Defaults to the package data dir (`LOOPS_DATA_DIR` or `~/.hasna/loops`). */ dataDir?: string; statePath?: string; outboxPath?: string; /** Defaults to `$LOOPS_RUNNER_NOTIFIER_CMD` when unset here. */ notifierCommand?: string; runnerId?: string; now?: () => Date; /** Structured journal sink; defaults to `console.error`. */ journal?: (line: string) => void; /** Notifier launcher; defaults to a detached `spawn` whose failures are swallowed. */ spawnNotifier?: (command: string, payload: string) => void; } export declare function runnerEpisodesStatePath(dataDirValue: string): string; export declare function runnerEventsOutboxPath(dataDirValue: string): string; export interface StateLock { /** The open writable fd; ownership of it passes to the caller (releaseStateLock closes it). */ fd: number; /** Unique per-acquisition identity, written into the lock file. */ token: string; } /** * Create the state lock with O_EXCL and stamp it with our token. Returns * undefined on contention (bounded retries) or an unexpected failure. The * token is what makes release ownership-checked (finding F5): only the holder * whose token still sits in the file may delete it. */ export declare function acquireStateLock(lockPath: string): StateLock | undefined; /** * Release the state lock (finding F5): delete the lock file only if it still * holds OUR token. A lock file holding a different token was created by a * successor after a stale takeover — deleting it would open a concurrent * entry while that successor is mid-critical-section. A missing file (the * takeover already renamed ours aside) is nothing to do. */ export declare function releaseStateLock(lockPath: string, fd: number, token: string): void; export declare function createRunnerEpisodeRecorder(opts?: RunnerEpisodeRecorderOptions): RunnerEpisodeRecorder;