import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider'; import type { HostedSpawnRequest, ManagedNodeInfo, NodeStateListener } from '../orchestrator/types.js'; import type { ClaimAddressSettings } from '../installer/config.js'; import { type ClaimDetails, type HostedNodeAddressSource } from './claim-details.js'; import type { HostedNodeStore } from './hosted-node-store.js'; import type { HostedNodeChange, HostedNodeChangeListener, HostedNodeStatus, HostedNodeView } from './types.js'; /** * Age after which a record still stuck in `spawning` is marked `error` and its * child, if any, reclaimed: the host wrote the row before starting the child, and * a crash/kill in that window (or during the spawn itself) leaves nothing to * advance it across a restart. A real spawn (identity key read/gen, port * allocation, process spawn) completes in well under a second normally, so 5 * minutes is generously past "any plausible spawn" without risking a false reap * of one still genuinely in flight under heavy load. */ export declare const HOSTED_NODE_SPAWNING_TTL_MS: number; /** How often the stuck-`spawning` reap runs while cadre-host is up. 5 minutes. */ export declare const HOSTED_NODE_REAP_SWEEP_MS: number; /** * Outcome of {@link HostedNodeService.respawn}: * * - `respawned` — a new child is up and the record now names its handles. * - `abandoned` — the record was removed, or went `error`, while the spawn was in * flight. The ending wins: the record is left exactly as the ending wrote it and * the new child is cleaned up. `status` is the status that won, absent when the * row is gone entirely. */ export type RespawnResult = { outcome: 'respawned'; node: HostedNodeView; } | { outcome: 'abandoned'; status?: HostedNodeStatus; }; /** Options for {@link HostedNodeService.respawn}. */ export interface RespawnOptions { /** * Whether this respawn spends one of the record's attempts. Default true: a crash * respawn does, since the supervisor's backoff and give-up read the count. A * deliberate restart of a running node (`HostedNodeSupervisor.restart`) passes false * and leaves both the counters and `updatedAt` alone. */ countAttempt?: boolean; } /** * Orchestrator capabilities the service needs beyond the base `Orchestrator`. * `createContainer` takes the claim secret. The rest are optional — only * `HostProcessOrchestrator` implements them; absent on a test double or a future * orchestrator, the cleanup degrades to terminalizing the record with nothing to * reclaim, and the claim details carry LAN addresses only. */ export interface HostedNodeOrchestrator extends Orchestrator { createContainer(request: HostedSpawnRequest): Promise; /** * Resolve a spawn's friendly containerId (== a hosted node's id) back to its * current dockerId, so a stuck-`spawning` reap can find and reclaim a child that * was actually spawned before the host died. */ resolveDockerId?(containerId: string): string | undefined; /** * Remove the working directory a spawn created for `containerId` when no handle * owns it — the cleanup of last resort for a record that never got a `dockerId`. * Returns whether anything was removed. Refuses when a handle still resolves * (that child's directory belongs to `removeContainer`). */ reclaimWorkdir?(containerId: string): boolean; /** The node's handle, for the ports its public addresses are built from. */ getNode?(idOrDockerId: string): ManagedNodeInfo | undefined; /** The exit/start signal the status watcher follows. */ onStateChange?(listener: NodeStateListener): () => void; } /** What `join` takes: nothing for a node waiting to be claimed, or the invitation the node redeems. */ export interface JoinOptions { /** An encoded cadre invitation (`encodeCadreInvitation`), as the owner's app copies it. */ invitation?: string; } /** Constructor options. */ export interface HostedNodeServiceOptions { /** Orchestrator that spawns/stops the hosted child processes. */ orchestrator: HostedNodeOrchestrator; /** Persistent record store (`hosted-nodes.json`). */ store: HostedNodeStore; /** The NAT layer, for the node's public addresses and reachability verdict. */ addresses: HostedNodeAddressSource; /** `host.config.json`'s `claimAddresses`, read on every claim-code request so an edit applies at once. */ claimAddressSettings?: () => ClaimAddressSettings | undefined; /** The machine's primary address, the `lan: 'auto'` fallback when no UPnP gateway answered. */ primaryLan?: () => Promise; /** Clock override for tests. */ now?: () => Date; } /** * HostedNodeService — the host's one action, "Join a cadre", and what follows from it. * * `join` starts a child waiting to be claimed, or one that redeems a cadre invitation; * `claimDetails` reads a waiting node's addresses and builds the QR payload; the status * watcher ({@link startWatching}) notices the claim or the redemption's outcome, and the * node's liveness; `respawn` brings a dead child back as the same node with the same * secret and ports; `retry` starts an invitation node again after no member could be * reached; `remove` and `reset` end it. The host contributes capacity only: it never * holds an owner key. * * **An ending that lands mid-operation wins.** Spawning a child takes seconds, and * a `remove` can land inside that window. `hosted-nodes.json` is written a whole row * at a time, so every long operation re-reads the record after its wait, decides * against what is actually stored, and merges forward only the fields it itself * produced — never the entry-time copy. * * **Respawns run one at a time** ({@link serializeRespawns}): the supervisor's passes and * restarts and `retry` share one queue, since two overlapping respawns of one id both spawn * a child and the second drops the first's orchestrator handle. */ export declare class HostedNodeService { private readonly orchestrator; private readonly store; private readonly addresses; private readonly claimAddressSettings; private readonly primaryLan; private readonly now; private readonly listeners; private readonly watcher; /** Serialization tail for `join`, so two joins at once cannot interleave their store writes. */ private joinTail; /** Serialization tail for every respawn — see {@link serializeRespawns}. */ private respawnTail; constructor(opts: HostedNodeServiceOptions); /** Register a change listener (the server publishes each change as an SSE event). Returns an unsubscribe fn. */ onChange(listener: HostedNodeChangeListener): () => void; /** * Report a change to the listeners. Public for the supervisor's give-up, which * writes the record itself; a listener that throws is logged, never propagated. */ emit(change: HostedNodeChange): void; /** Start following every node's `/status` (claims and liveness). Idempotent. */ startWatching(): void; stopWatching(): void; /** One status pass now, awaited — for tests and callers that need the result before going on. */ pollStatuses(): Promise; /** * Start a node: write the record, spawn the child, and record its handles. Without an * invitation the node waits to be claimed — a fresh claim secret, the placeholder * party, `unclaimed`. With one it redeems the invitation — the invitation's party, * `joining` — and an invitation that does not decode is `invalid_request` before * anything is written. On any orchestrator failure the reserved resources are * reclaimed and the record is marked `error`. Resolves once the child is spawned; its * `/status` may take a few seconds more, which `claimDetails` reports as * `node_unavailable`. */ join(opts?: JoinOptions): Promise; private joinLocked; /** * Mark a still-`spawning` record as failed — and ONLY such a record. A row that * was removed while the spawn was in flight must not be recreated. * * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a * spawn failure, and a store error here must not mask the error being reported * to the caller. */ private markSpawnFailed; /** * What the QR code carries for a node waiting to be claimed: its peer id, the * addresses a phone may dial it at, and its claim secret — the only place the * secret ever leaves the host. `409 invalid_state` unless the node is `unclaimed`; * `503 node_unavailable` until the child's `/status` answers (callers poll). */ claimDetails(id: string): Promise; /** Cache the peer id on the record, once. Best-effort: the claim details do not depend on it. */ private recordPeerId; /** * Re-spawn a node that is no longer running, replaying the record's party and * claim secret, or its invitation while it is still `joining`. The node keeps its * workdir — and with it its identity key and node-local stores — and comes back on * the same ports, so it is the *same* peer at the same addresses: a QR code already * shown stays valid, and a joined node's cadre finds it where it was. Keeping the * secret after the claim is deliberate: `cadre-cli` honours `claim.json` and, with the * secret still set, answers a rival `already-claimed`. A joined invitation node comes * back without its invitation ({@link spawnRequestFor} says why). * * **Status is deliberately unchanged.** A `joined` node stays `joined`; an * `unclaimed` one keeps waiting, a `joining` one redeems again. `connected` is * dropped until the watcher reads the new child. * * **An ending that lands mid-spawn wins.** The record is re-read after the * orchestrator returns; if it was removed (or went `error`) while the child was * starting, that write stands and the new child is abandoned — see * {@link abandonRespawn}. An abandoned respawn is a skip, not a failure, so a * sweep over the store keeps going. Orchestrator failures DO throw; the caller * owns backoff/give-up. * * Not serialized itself: callers run it on {@link serializeRespawns}. Two overlapping * calls for the same id both spawn a child, and the second spawn drops the first's * orchestrator handle — leaving an unmanaged process. */ respawn(id: string, opts?: RespawnOptions): Promise; /** * Start an invitation node again after no member of its cadre could be reached: the * `error` record goes back to `joining` and is respawned with its invitation, without * spending a respawn attempt. On the respawn queue, so it never overlaps the * supervisor's handling of the same node. `409 invalid_state` unless the node is `error` * with `retryable` — a refused invitation stays refused, and the supervisor's give-up and * the other failures are a reset's. */ retry(id: string): Promise; private retryLocked; /** * Give up on a respawn whose record ended while the child was starting: stop * the new child and, unless the record went `error`, reclaim it. * * Reclaim, not merely stop. The removal's own cleanup already ran and found * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle` * *while* spawning, so by the time the concurrent `remove` reached its * stop/reclaim the old handle was gone and both calls were swallowed as * best-effort no-ops. The new handle is now the only thing holding that * spawn's ports and workdir, so stopping alone would leak both. * * `error` is the deliberate exception: `removeContainer` deletes the workdir, * and `HostedNodeSupervisor.giveUp` keeps it on purpose so the identity key the * node's cadre approved survives. Since both spawns share one workdir * (`/`), reclaiming here would delete exactly what * `giveUp` meant to keep. * * The `error` record is then pointed at the new child, the one that exists, so a * later `remove` reclaims its ports and workdir. `giveUp` runs on the respawn queue * and cannot overlap a respawn; the watcher's write of a failed invitation can, when * a poll of the old child answers while the new one is starting. */ private abandonRespawn; /** * Remove a hosted node: delete the row FIRST, then stop and reclaim the child * (its workdir, identity key and data go with it). The node's cadre keeps the * node's row until its owner removes it there. * * Order matters: stopping the child fires the orchestrator's `onStateChange`, * and the supervisor's exit-triggered pass must find no record — otherwise it * observes "node gone, record still `joined`" and brings back a node the admin * just removed. The tradeoff: a crash between the delete and the stop leaves a * running child no record names, which `remove` on its id still reclaims (below). * * A handle no record names — `hosted-nodes.json` lost, or a handle an older * build persisted — is removed the same way, so `/api/nodes` cannot list a node * for good that nothing can end. */ remove(id: string): Promise; private removeOrphanHandle; /** * Remove a node and start a fresh one waiting to be claimed, with a new secret and * identity. For the case where someone else photographed the QR code and claimed the * node first, and for an `error` node the admin wants to try again. An invitation node * is replaced by one waiting to be claimed too: its invitation may be spent, and the * admin can join with a fresh one instead. */ reset(id: string): Promise; /** * Mark records stuck in `spawning` past `ttlMs` as `error` and reclaim whatever * the orchestrator can still find for them — the record is written before the * child spawn starts, so a host crash/kill anywhere in that window (including * mid-spawn) leaves nothing to advance it across a restart. Run once at * startup and on a periodic sweep. Age is measured from `updatedAt`, which for * a genuinely-stuck row equals `createdAt` — `joinLocked` never touches a * `spawning` record again except to advance it out of this predicate. * Best-effort per record: a failed reap is logged and the sweep continues. * Returns the reaped ids. */ reapStuckSpawning(ttlMs?: number): Promise; /** * Terminalize a stuck-`spawning` record as `error` and reclaim any child the * orchestrator can still find for it. Status is written FIRST (same ordering * rule as {@link remove}): reclaiming fires `onStateChange`, and anything * listening must already see a terminal record. * * NOTE: the stop only reaches a child the orchestrator has a handle for. * `HostProcessOrchestrator.launchChild` spawns the process and persists the * handle in the same synchronous step, so the only child this misses is one * whose host died between the OS spawn and that write — an orphan *process* * this reap terminalizes the record for but cannot kill (its ports stay held * until a reboot). Its working directory is reclaimed by container name when * no handle resolves, which also covers the far commoner case of a host that * died before any child existed at all. */ private reclaimStuckSpawning; /** One node (secret stripped), or undefined when unknown. */ get(id: string): HostedNodeView | undefined; /** Every node (secret stripped). */ list(): HostedNodeView[]; private requireNode; /** Best-effort stop; logs but never throws (cleanup path). */ private safeStop; /** * Persist a failed respawn's attempt counters (none for an uncounted restart) — * plus, when the spawn itself succeeded and it was the record write that failed, * the new child's handles. Merged onto whatever is on disk now, never written * wholesale: the caller's copy predates the orchestrator round-trip. * * `status` and `updatedAt` are deliberately untouched — the attempt did not succeed. * * Recording `spawned` is what keeps the record pointing at the child that * actually exists: `createContainer` already dropped the handle the record * previously named, so leaving the old `dockerId` there would send every later * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's * workdir on disk forever. * * Best-effort: we are already unwinding a respawn failure, and a store error * here must not mask it. Losing the write costs the caller one extra attempt * before backoff; the supervisor's next pass re-reads the record, finds the * named child not running, and respawns again — which drops whatever handle is * current, so the ports self-heal either way. */ private storeRespawnAttempt; /** Best-effort remove; logs but never throws (cleanup path). */ private safeReclaim; /** * Best-effort removal of the working directory of a node whose spawn never * produced a `dockerId`; logs but never throws (cleanup path). A no-op on an * orchestrator without the capability, and on one that refuses because a live * handle still owns the directory. */ private safeReclaimWorkdir; /** Run `fn` after any in-flight join. */ private serializeJoin; /** * Run `fn` after every respawn already queued. Every caller of {@link respawn} goes * through here: the supervisor's reconcile passes and restarts, and {@link retry}. */ serializeRespawns(fn: () => Promise): Promise; }