/** * LongRunningSupervisor — Option C: producer as a long-lived child process. * * Design choice: sibling class (not a mode flag on Supervisor) because the * lifecycle model is fundamentally different — no per-tick scheduling, no * tick overlap guard, no tickSeconds semantics. Sharing a "mode" enum would * require branching half the class; a sibling keeps both implementations clean * and testable in isolation. Both import the kill/reap machinery from * supervisor-utils.ts — no duplication. * * Key behaviours (Option C spec): * 1. Spawn once; the child owns its loop. Restart on ANY exit. * 2. Liveness = exit (instant detection). Restart with capped exponential * backoff. Rapid crash-loop guard: degrade after N fast failures, keep * retrying at max backoff — never give up silently, never hot-loop. * 3. Graceful stop(): cancel pending restart → SIGTERM → bounded grace * (drain window, killGraceMs) → SIGKILL → reap descendants. Resolves only * when process tree is dead. * 4. Startup orphan sweep: static sweepOrphans() re-exported here for * convenience; the marker env var is stamped on every spawn. * 5. Staggered start: configurable initialDelayMs jitter before first spawn. * 6. Pluggable silence-kill: optional lastAliveAt() callback + maxSilenceMs * threshold. Default OFF. The heartbeat protocol is out of scope. * 7. Observable: typed events for every lifecycle transition. */ import { EventEmitter } from "node:events"; import type { Logger } from "../../utils/logger.js"; /** Configuration for what to run and how. */ export interface LongRunningSpec { /** Executable path. */ command: string; /** Arguments passed to the command. */ args: string[]; /** Additional environment variables merged with process.env. */ env: Record; /** Base restart delay in ms (default: 500). */ restartBackoffMs?: number; /** Backoff cap in ms (default: 30_000). */ maxBackoffMs?: number; /** * Window in ms in which a child must survive to be considered stable * (backoff resets after this duration). Default: 10_000. */ stableUptimeMs?: number; /** * Number of rapid exits (within stableUptimeMs) before entering degraded * state. Default: 5. After this many rapid failures the supervisor keeps * retrying at maxBackoffMs but marks itself degraded. */ degradeAfterRapidFailures?: number; /** * Bounded drain window (ms): how long to wait after SIGTERM for the child to * finish in-flight work and exit cleanly before escalating to SIGKILL. This * IS the drain phase — there is exactly one post-SIGTERM grace window, owned * by the hardened shared killChild(). Default: 1_000. */ killGraceMs?: number; /** * Delay the first spawn by this many ms. Use when many supervisors start * concurrently to avoid a thundering herd. Default: 0. */ initialDelayMs?: number; /** * Optional liveness source. When provided along with maxSilenceMs, the * supervisor polls this callback; if it returns a timestamp older than * maxSilenceMs, the child is killed (triggering the normal restart path). * DEFAULT: OFF (both fields must be set to enable). */ lastAliveAt?: () => number | null; /** * Maximum age (ms) of the lastAliveAt timestamp before the child is * considered silent and killed. Default: undefined (feature disabled). */ maxSilenceMs?: number; /** * How often to poll lastAliveAt in ms. Default: 5_000. */ silenceCheckIntervalMs?: number; /** * Extra structured fields stamped on every forwarded child stdout/stderr line. * The supervisor's own identity fields (supervisorId/runId/pid/stream) always * win on a key collision. Vendor-neutral: the caller owns the key names. * Read at forward time, so a mutation of `supervisor.spec.logAttributes` takes * effect on the next line. * * The supervisor is not the last writer: the logger layers the async-context * identity bag on top of any record's attributes, so a key that bag also * carries is decided by the context the spawn inherited, not by this spec. * Spawn outside a per-flow context (the launcher does) to keep these values. */ logAttributes?: Record; /** * Optional stable marker stamped as SENPI_SUPERVISED on every spawn. * When set, the supervisor stamps this value instead of its ephemeral * supervisorId — enabling a targeted boot sweepOrphans(runtimeId) that * kills only this runtime's children without touching neighbours. * Default: undefined → stamps supervisorId (back-compat with existing callers). */ marker?: string; } export type LongRunningStatus = "healthy" | "degraded"; /** Typed lifecycle events emitted on the "event" channel. */ export type LongRunningEvent = { type: "spawned"; pid: number; runId: string; attempt: number; } | { type: "exited"; pid: number; exitCode: number | null; signal?: string; durationMs: number; runId: string; /** The spawn/start exception when the child never ran (ENOENT, a synchronous spawn throw). */ error?: unknown; } | { type: "restarting"; runId: string; backoffMs: number; attempt: number; } | { type: "degraded"; reason: string; rapidFailures: number; } | { type: "recovered"; } | { type: "stopped"; wasDegraded: boolean; } | { type: "silence-killed"; pid: number; silenceMs: number; runId: string; } | { type: "swept-orphan"; pid: number; supervisorId: string; }; export declare class LongRunningSupervisor extends EventEmitter { readonly supervisorId: string; spec: LongRunningSpec; private readonly logger; private status; /** True once start() has been called. */ private started; /** True once stop() has been called — blocks any further spawning. */ private stopped; /** The currently live child process. */ private activeChild; /** Pending restart-backoff timer handle + canceller. */ private backoffTimer; private backoffResolve; /** Pending initial-delay timer. */ private initialDelayTimer; private initialDelayResolve; /** Silence-kill polling timer. */ private silenceTimer; /** * Stable-uptime timer: fires once the current child has been up for * stableUptimeMs, resetting the crash-loop counter and recovering from * degraded WITHOUT waiting for the next exit. Without it, a child that * stabilizes and runs forever would leave getStatus() === "degraded" * indefinitely — a standing false alarm. */ private stableTimer; /** * The promise that resolves when the run loop exits (either naturally via * stop() or if an unrecoverable internal error surfaces). */ private runLoopPromise; /** Count of rapid successive exits for the crash-loop guard. */ private rapidFailures; /** Timestamp when the current run was spawned. */ private runStartedAt; constructor(spec: LongRunningSpec, logger: Logger); getStatus(): LongRunningStatus; /** * Spawn the child and begin the restart loop. Fire-and-forget — returns * immediately. The loop runs as a background promise tracked via * runLoopPromise (awaited by stop()). */ start(): void; /** * Graceful shutdown cascade (hard requirement): * 1. Cancel any pending restart timer. * 2. Cancel any pending initial-delay timer. * 3. Stop the silence-kill poller. * 4. SIGTERM the active child. * 5. Wait for the bounded drain window (killGraceMs). * 6. SIGKILL any survivor + reap descendants. * 7. Await the run loop to fully unwind. * 8. Emit "stopped". * * Idempotent. Resolves only when the process tree is dead. */ stop(): Promise; private runLoop; private spawnChild; /** * Wait for the child to exit. Returns immediately if stop() has been * called (the kill is handled by stop() itself). */ private waitForExit; private startSilencePoller; private stopSilencePoller; private computeBackoff; private backoff; private cancelBackoff; private delayInitial; private cancelInitialDelay; /** * Arm a one-shot timer that marks the run stable after stableUptimeMs of * continuous uptime. Without it, recovery from degraded would only happen on * the NEXT exit — a child that stabilizes and runs forever would leave the * supervisor reporting "degraded" indefinitely. */ private startStableTimer; private clearStableTimer; /** Reset the crash-loop counter; transition degraded → healthy once. */ private markStable; private recordFailure; private emitRestarting; /** * Emit on the "event" channel, isolated from throwing listeners. * * EventEmitter.emit is SYNCHRONOUS — an exception from a listener would * otherwise unwind supervisor internals mid-mutation: it can prevent * waitForExit's resolve() (wedging the run loop and deadlocking stop()), * reject runLoopPromise before activeChild is assigned (leaking a live * child), or escape a timer callback as an uncaught exception (crashing the * host). Listener errors are logged and swallowed; the supervisor's own * state machine must never depend on listener good behaviour. */ private safeEmit; } //# sourceMappingURL=long-running-supervisor.d.ts.map