import type { Orchestrator } from '@serfab/cadre-provider'; import type { ManagedNodeInfo, NodeStateListener } from '../orchestrator/types.js'; import type { HostedNodeService } from './hosted-node-service.js'; import type { HostedNodeStore } from './hosted-node-store.js'; /** * Unit of the doubling respawn backoff: the wait after `n` consecutive failed * attempts is `base * 2^n`, capped at {@link HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS}. * So the first retry waits 10s (one attempt is already recorded by then), the * second 20s, and so on. A node with no recorded attempt is respawned * immediately. */ export declare const HOSTED_NODE_RESPAWN_BACKOFF_BASE_MS = 5000; /** * Ceiling on the doubling respawn backoff. 5 minutes. * * NOTE: not reachable at the current {@link HOSTED_NODE_RESPAWN_MAX_ATTEMPTS} of 5 * — the longest wait actually served is the one before the 5th attempt, * `5s * 2^4` = 80s, and once that attempt fails or its node dies the supervisor * gives up rather than waiting again. The cap only starts clamping * once a record can reach 6 recorded attempts (`5s * 2^6` = 320s), so raising * the attempt cap without revisiting this one changes nothing. */ export declare const HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS: number; /** * Consecutive respawn attempts after which the host stops trying and marks the * node `error` — a node that will not stay up is a host-side fault its cadre * cannot see, so surface it instead of spinning forever. * * Every attempt counts, whether its spawn failed or succeeded: a node that spawns * and then dies before {@link HOSTED_NODE_RESPAWN_HEALTHY_MS} refills the budget is * as much a crash loop as one that never spawns. Give-up happens on whichever * comes first — an attempt that throws with the cap reached, or a pass that finds * the node down with the cap already reached. */ export declare const HOSTED_NODE_RESPAWN_MAX_ATTEMPTS = 5; /** * How long a respawned node must stay up before its attempt budget is refilled. * The last respawn produced something that survived, so the next crash starts * from a fresh budget rather than inheriting an old crash loop's count. */ export declare const HOSTED_NODE_RESPAWN_HEALTHY_MS: number; /** How often the supervisor sweep runs while cadre-host is up. 1 minute. */ export declare const HOSTED_NODE_RESPAWN_SWEEP_MS = 60000; /** * The orchestrator surface the supervisor needs: the shared `Orchestrator` * (for `isRunning` / `stopContainer`) plus cadre-host's own exit-event * subscription and node list, which are not part of that cross-package * interface. `HostProcessOrchestrator` satisfies this. */ export interface SupervisedOrchestrator extends Orchestrator { onStateChange(listener: NodeStateListener): () => void; listNodes(): ManagedNodeInfo[]; } /** Constructor options. */ export interface HostedNodeSupervisorOptions { /** Lifecycle service — the supervisor drives its `respawn`. */ service: HostedNodeService; /** Persistent record store — the candidate list and the give-up write. */ store: HostedNodeStore; /** Orchestrator that owns the hosted child processes. */ orchestrator: SupervisedOrchestrator; /** Clock override for tests. */ now?: () => Date; } /** * HostedNodeSupervisor — owns the invariant *an `unclaimed`, `joining` or `joined` * node is expected to be running*. * * Nothing else re-spawns a hosted node: `/api/nodes` is read-only. So without * this class a crashed, OOM-killed, or reboot-killed node stays dead with its * record still reading `joined`, silently costing its cadre a node. * * One code path ({@link reconcile}) behind three triggers: * * - **host startup** — one pass after `orchestrator.init()`, which catches every * node that died while the host was down; * - **exit signal** — the orchestrator's `onStateChange`, so a crash is noticed * in milliseconds rather than at the next sweep; * - **periodic sweep** — the backstop for deaths no exit event covers (host was * down, listener missed, orchestrator re-attached to an already-dead pid). * * Passes are serialized: `HostedNodeService.respawn` is not itself serialized, and * two overlapping respawns of one id both spawn a child while the second drops * the first's orchestrator handle (see its docstring). Every pass, and a deliberate * {@link restart}, runs on the service's respawn queue * (`HostedNodeService.serializeRespawns`), which the service's Retry shares. * * The supervisor reconciles records to handles, never handles to records: a * handle no record names (`hosted-nodes.json` lost, or a handle an older build * persisted) is logged at startup and left running, since stopping it could end * a node that is serving its cadre. `cadre-host node remove ` reclaims it. */ export declare class HostedNodeSupervisor { private readonly service; private readonly store; private readonly orchestrator; private readonly now; private timer?; private unsubscribe?; /** At most one exit-triggered pass queued at a time (crash storms coalesce). */ private exitPassQueued; /** Set by `stop()` so an in-flight pass abandons its remaining records. */ private disposed; constructor(opts: HostedNodeSupervisorOptions); /** * One reconcile pass over the store. Serialized against every other pass, so * a caller may get a queued pass rather than an immediate one. Returns the ids * it respawned in *its own* pass. */ reconcile(): Promise; /** * Stop a running node and start it again, for a change the node picks up * only at start — its public addresses (`NatService.onNodeAddressesStale`). * Serialized with the reconcile passes, so it never overlaps a respawn or a * give-up of the same record, and it re-reads the record first: a record that * is no longer supervised, has no handle, or whose node is not * running is left alone (a node that is down is the crash path's). * * Does not spend the record's respawn budget. A restart whose respawn throws * rejects; the node is then down, and the exit event its stop raised has queued * a reconcile pass that takes it over through the crash path, with its counting. */ restart(id: string): Promise; /** * Subscribe to child exits, start the periodic sweep, and run the startup * pass. Idempotent — a second call while started is a no-op. */ start(): void; /** * Stop the timer and unsubscribe. A pass already in flight abandons its * remaining records rather than being awaited — every record it has not * reached is picked up by the next host start's startup pass. */ stop(): void; /** * Name every orchestrator handle that no record claims, once at startup. Not * reclaimed (see the class docstring): the admin ends it with * `cadre-host node remove `. */ private logOrphanHandles; /** Fire-and-forget pass for the timer / exit / startup triggers. */ private sweep; /** * Exit-event trigger. Deliberately cheap: it does not map the handle back to a * record (that needs a store read per event), it just asks for a pass. */ private onNodeState; /** The unserialized body of one pass. */ private reconcileOnce; private restartOnce; /** Reconcile one record. Returns its id when this pass respawned it. */ private reconcileOne; /** * Whether enough time has passed since the last attempt. A record with no * attempt history (or an unparsable timestamp — never wedge a node on bad * data) is eligible immediately. */ private backoffElapsed; /** * Clear the attempt count once a respawned node has been up long enough to * count as healthy, so the next crash gets a full budget instead of inheriting * the previous crash loop's. */ private refillBudgetIfHealthy; /** * Whether a record carries a crash-loop count old enough to clear. An * unparsable timestamp is treated as not-due — unlike the backoff check, * being wrong here would erase real attempt history rather than free a node. */ private budgetRefillDue; /** * One respawn attempt, with the give-up check on failure. (The check for an * attempt that spawned but then died lives in {@link reconcileOne}.) */ private attemptRespawn; /** * Stop trying: mark the node `error` and stop whatever child the record * still names. * * The record is written BEFORE the stop for the same reason `remove` does * it in that order — the stop fires `onStateChange`, and a pass triggered by * it must not see "node gone, record still `joined`" and start respawning * again. * * The child is stopped but NOT removed: `removeContainer` deletes the workdir, * and the workdir holds the identity key the node's cadre approved. A later * `remove()` (or `reset()`) on an `error` record reclaims it. * * NOTE: an `error`-after-give-up record therefore keeps its workdir until * someone removes it. If stale `error` workdirs ever pile up on real hosts, * extend the reap sweep in `hosted-node-service.ts` to cover them. */ private giveUp; }