import * as ManagerPB from "@norskvideo/norsk-api/lib/manager_pb"; import { Clock } from "./clock"; import { Bundle as TypedBundle, BundleId, NodeInventory as TypedNodeInventory, NodeService, ServiceId } from "./types"; import { ValidationError, ValidationResult } from "./validation"; import { PlacementFailureReason, PlacementPool, PlacementResult, PlacementTrace, ResilienceFlag } from "./placement"; import { WarmNodeTransport } from "./warmAdapter"; export { migrationTargetRole, newestHistoryEntryForRole, readApplySummary } from "./migrationSaga"; import { NorskManager, JobId, JobInfo, JobWithHistory, NodeId, NodeMetadata, Role, RunningNodeMetadata } from "@norskvideo/norsk-manager-sdk"; import type * as ManagerSdk from "@norskvideo/norsk-manager-sdk"; export type NodeState = "starting" | "running" | "stopping" | "stopped" | "terminating" | "terminated"; export type NodeSummary = { nodeId: NodeId; nodeMetadata?: NodeMetadata; runningNodeMetadata?: RunningNodeMetadata; nodeState: NodeState; lastActivity: Date; provisional?: { jobId: JobId; role: Role; }; }; export interface AutoSettings { url?: string; /** * Abort the connection / run. Aborting before `run()` connects rejects it * (so a connect-with-timeout retry loop can give up cleanly without * leaving an abandoned SDK connection alive); aborting a running * AutoManager tears down its Manager connection, ending the run loop. */ signal?: AbortSignal; /** Seconds-from-now window for jobs to be considered pending in the eventStream. */ pendingWindow: number; /** Seconds stopped nodes persist before being terminated. */ removeStoppedNodesAfter: number; /** Configured pools; a bundle names one, placement walks its tiers. */ placementPools: PlacementPool[]; /** * Window (ms) for counting jobRejected events per-node. Default 5_000. * Once `rejectionEscalationCount` rejections fall inside this window the * node is marked "unhealthy" and excluded from placement until either a * job reaches `running` on it or `unhealthyCooldownMs` elapses. (Routine * inventory heartbeats deliberately do NOT clear the mark.) */ rejectionBackoffMs?: number; /** Default 3. */ rejectionEscalationCount?: number; /** * How long (ms) an "unhealthy" mark excludes a node from placement * before it is allowed to try again. Default 600_000 (10 min). A job * reaching `running` on the node clears the mark early. */ unhealthyCooldownMs?: number; /** * After a node fails (`nodeStopping`), how long to keep it excluded from * placement. Default 30_000 ms. Expires on read; a job reaching * `running` on the node clears it early. */ failureBackoffMs?: number; /** * Phase C2: per-replica restart-rate ceiling on the recovery path. * When a replica's restart count in the trailing hour exceeds this, * AutoManager stops re-placing it (and every other replica of the * same bundle), marks the bundle broken, and fires `onBundleBroken`. * Cleared when the bundle is updated or deleted, or when a fresh * `jobUpdated` arrives for the broken job. Default 6 (per design). */ maxRestartsPerHour?: number; /** * Phase C3: node startup grace. If a node stays in `nodeStarting` * for longer than this, AutoManager terminates it — Manager emits a * `nodeStopping` and the existing policy-aware recovery path replaces * the job. Default 120_000 ms. * * (In v1, "job in starting" and "node in starting" collapse onto * the same signal because `running → ready` is auto-marked. The * design's distinction matters when explicit readiness lands.) */ jobStartupGraceMs?: number; /** * Phase C4 — per-policy recovery tunables. When set, takes precedence * over the flat fields above. Only the two re-placing policies appear: * `drop` never restarts, so a restart ceiling is meaningless for it. * Whether a recovering job prefers or avoids hot spares follows from * the policy itself, not from config. Omitted entries fall back to the * flat top-level fields, then to defaults. */ failurePolicies?: FailurePolicyConfig; /** * Phase C4 — placement / recovery tunables grouped into one typed * object. Same precedence as `failurePolicies`: object field beats flat field * beats default. The flat fields above are preserved for one * release; new code should prefer this shape. */ placement?: PlacementConfig; /** * Phase C6 — hot-spare reconciliation configuration. One entry per * pool (plus its node spec). The reconciler runs on node lifecycle * events and on a periodic tick: counts live spares per entry, * provisions short, terminates excess. Empty/omitted = no spares. */ hotSpares?: HotSpareConfig[]; /** * Phase F — NodeService configurations. Each entry describes a * long-running service that consumes capabilities (typically * exclusive hardware) and advertises new ones for sibling jobs * to consume. AutoManager's reconciler places eager services on * every eligible node and tears them down when a node loses * eligibility. * * Worker-side capability advertisement is required for sibling * jobs to actually see the provided capabilities — see * `§Worker-side work` in the design doc. */ nodeServices?: NodeService[]; /** * Feature fences. A non-empty `nodeServices` is refused at boot unless * `experimental.nodeServices` is true: what exists is a working F1 (eager * placement on every eligible node) with none of the F2+ pieces — network * placement, identity across restarts, failure policy, `provides` into * inventory — so a yaml that happens to carry `nodeServices` must not * place a service on every node by accident. See norsk-mgr * `docs/internal/design/node-services.md` §"Meanwhile: fence what exists". */ experimental?: { nodeServices?: boolean; }; /** * Phase F — how often the NodeService reconciler's periodic tick * fires. Event-driven reconciles handle most cases; the tick is a * safety net. Default 60_000 ms. */ nodeServiceReconcileIntervalMs?: number; /** * Phase C6 — how often the spare reconciler's periodic tick fires. * Event-driven reconciles handle most cases; the tick is a safety * net for missed signals. Default 60_000 ms. */ hotSpareReconcileIntervalMs?: number; /** * Phase G — which container images each product template needs, keyed by * the template's registered name (`ProductTemplateRef.templateName`; a * ref without one keys by `productName`). Placement scores warmth as the * overlap between this set and what a node advertises holding, and the * spare reconciler prefetches it for a `warmFor` template. * * Supplied by the embedder because the template tar is opaque to Manager: * norsk-ctl reads the images out of the manifest it already parses. * Entries may be `repository:tag`, `repository@sha256:...` or both * (`repository:tag@sha256:...`) — a digest matches exactly, a tag is the * documented fallback when the digest is not known at placement time. * * `setTemplateImages` adds/replaces one entry at runtime; unknown * templates simply have no warmth signal (every node scores 0), which is * the pre-Phase-G behaviour. */ templateImages?: Record; /** * Phase G — override the G1 transport (`src/warmAdapter.ts`): the image * prefetch, the per-node idle-expiry RPC, and how a node's advertised * image set is read out of its inventory. Production leaves this unset * and gets the duck-typed binding to the Manager SDK; tests and * embedders ahead of the wire supply their own. */ warmNodes?: Partial; /** * Phase G — one prefetch batch has settled on a node, one result per * requested ref. A `ok: false` entry is worth surfacing: that spare * stays cold for whatever template needed the ref. */ onImagesPrefetched?: (nodeId: NodeId, results: { imageRef: string; digest: string; ok: boolean; error: string; }[]) => void; /** * Phase E — per-phase deadline configuration for in-flight * migrations. If a migration sits in a phase longer than the * configured window, AutoManager auto-aborts with reason * `phase timeout: `. Operators can still abort manually via * `abortMigration` before the deadline fires. */ migration?: MigrationConfig; onError?: (err: unknown) => void; onNodeStarting?: (node: NodeId) => void; onNodeStarted?: (node: NodeSummary) => void; /** Fired when the daemon emits a NodeStopping event — the node is * going away but may not have fully terminated yet. For cluster * workers Ctrl-C'd at the keyboard this is the signal that fires * (NodeStopped/NodeTerminated come later, after the daemon's * keepalive timeout or cleanup loop). Consumers wanting prompt * "worker dropped" UI updates listen here. */ onNodeStopping?: (nodeId: NodeId) => void; onNodeStopped?: (nodeId: NodeId) => void; /** * Fired when THIS AutoManager issues a terminate for a node — from any * of its own paths: `terminateNode()` / `evictNode()` / a `drainNode()` * that emptied its node, the hot-spare * excess and orphan reapers, single-use release, startup-grace expiry * and the stopped-node sweep. Exactly once per node, however many of * those paths (or retries of one) issue the RPC. It fires when the * terminate is issued, not when the daemon confirms it: the daemon's * `NodeStopping` / `NodeStopped` follow when it acts. A node terminated * by anyone else (an operator going to Manager directly, the daemon's * own empty-node sweep) is reported ONLY through `onNodeStopping` / * `onNodeStopped` — this is not "a node was terminated" in general. */ onNodeTerminated?: (nodeId: NodeId) => void; onBundleUpdated?: (bundle: TypedBundle) => void; onBundleDeleted?: (bundleId: BundleId) => void; /** * Fired when the daemon appends a new audit-log entry (any source). * Pure pass-through of the SDK's `AuditLogEntry` — AutoManager doesn't * own audit semantics, it just forwards the live append so consumers * can tail the log without re-polling GetAuditLog. */ onAuditAppended?: (entry: ManagerSdk.AuditLogEntry) => void; /** * Fires when a node accumulates enough rejections within * `rejectionBackoffMs` to be marked unhealthy. Operators may want to * surface this in monitoring. */ onNodeUnhealthy?: (nodeId: NodeId) => void; /** * Fires when a previously-unhealthy node's exclusion is cleared — * either a job reached `running` on it, or its mark aged past * `unhealthyCooldownMs`. Lets a consumer mirroring the unhealthy set * keep its count accurate. */ onNodeHealthy?: (nodeId: NodeId) => void; /** * Fired after `jobMap` is updated in response to a `jobUpdated` or * `jobPending` event. Carries the full `JobWithHistory` so consumers * can render state transitions without a separate SDK round-trip. */ onJobUpdated?: (jobWithHistory: JobWithHistory) => void; /** * Fired for an informational job message from the worker — launch * progress (pulling-images / starting-containers / health), container * events, service lifecycle. Lets consumers surface live launch feedback * tied to a job without changing its placement state. */ onJobInfo?: (info: JobInfo) => void; /** * Fired after a job is removed from `jobMap` in response to a * `jobDeleted` or `jobOutOfWindow` event. */ onJobDeleted?: (jobId: JobId) => void; /** * Fired on every `jobRejected` event after AutoManager has updated * its inventory cache. Note the node is not necessarily marked * unhealthy by this single occurrence — see `onNodeUnhealthy` for * escalation. */ onJobRejected?: (info: JobRejectedInfo) => void; /** * Fires every time the placement engine runs, regardless of outcome. * Carries the full trace so consumers (e.g. an operator UI) can show * "why did my job land here?" without re-running placement * themselves. */ onPlacementDecision?: (info: PlacementDecisionInfo) => void; /** * Fires when the placement engine returns `kind: "failure"`. Today * the failure also surfaces via `onError` as a stringified message; * this typed callback is the recommended path. The two are not * mutually exclusive — `onError` is preserved for back-compat. */ onPlacementFailed?: (info: PlacementFailedInfo) => void; /** * Fired whenever the underlying gRPC channel changes connectivity * state. Maps the five gRPC connectivity states to the strings * `idle | connecting | ready | transientFailure | shutdown`. * Lets a consumer surface "connected / reconnecting / disconnected" * without holding a separate NorskManager handle. */ onConnectionStateChange?: (state: ConnectionState) => void; /** * Fired when a bundle's restart rate exceeds its `maxRestartsPerHour` * (or a runtime ends terminally: a `drop` policy, an invalid spec) and * AutoManager marks it broken. */ onBundleBroken?: (bundleId: BundleId, reason: string) => void; /** * Fired when a job fails and is dropped rather than restarted * (`onRuntimeFailure: drop`, a non-essential runtime past its ceiling, * or an invalid spec). */ onJobDropped?: (jobId: JobId) => void; /** * Phase-C: fired when a hot-spare pool's live spare count is below * its configured target. Declared now; not fired in pre-Phase-C * code. */ onSpareUnderflow?: (pool: string, current: number, target: number) => void; /** * Fired when a bundle's backup is best-effort placed inside a failure * domain its resilience policy asked to avoid (no compliant node had * capacity). The backup is running, but DR posture is reduced — surface * it so an operator can act / re-place when capacity frees. */ onResilienceDegraded?: (info: ResilienceDegradedInfo) => void; /** * Phase E — fired on every migration phase transition. Use for * telemetry / ops UIs. The payload is the post-transition state; * `phaseHistory[0]` is launch, `phaseHistory.at(-1)` is the current * phase at the time of the callback. */ onMigrationPhase?: (info: MigrationState) => void; /** Phase E — fired once when a migration reaches `done`. */ onMigrationCompleted?: (migrationId: MigrationId) => void; /** Phase E — fired once when a migration enters `aborted`. */ onMigrationAborted?: (migrationId: MigrationId, reason: string) => void; /** * Fired on every change to a node drain's state — a phase transition, * or a job's status within it. The payload is a snapshot; the same * state is readable at any time through `drainStatus(nodeId)`. */ onDrainProgress?: (state: DrainState) => void; /** * Fired once per `handleNodeInterruption` call, after the node is * cordoned and its drain started (or tightened). The payload is a * snapshot; `nodeInterruption(nodeId)` reads the current one. */ onNodeInterrupted?: (state: NodeInterruptionState) => void; /** Optional clock injection. Production unset (RealClock); tests pass MockClock. */ clock?: Clock; } export { NODE_SERVICE_ROLE } from "./nodeServices"; /** @public */ export type MigrationId = string; /** * How the two instances of a migrating job hand over. * - `overlap`: start the target, and once it is running stop the * source. Both run for a moment; no coordination with the job. * - `stopThenStart`: create the target without starting it * (`deferStart`), stop the source, then start the prepared target. * Never two running at once; a gap of one container start. * - `handover`: start the target and run the four-step handshake * (target ready → source stops output → target applies) before * stopping the source. The job owns the handover payload. * @public */ export type MigrationStrategy = "overlap" | "stopThenStart" | "handover"; /** * Phases, in order per strategy: * ``` * overlap: placing → launching → stoppingSource → done * stopThenStart: placing → launching → stoppingSource → startingTarget → done * handover: placing → launching → awaitingTargetReady → awaitingSourceStop * → applying → stoppingSource → done * any: … → aborted * ``` * @public */ export type MigrationPhase = "proposing" | "placing" | "launching" | "settingStateOnTarget" | "awaitingTargetReady" | "awaitingSourceStop" | "applying" | "stoppingSource" | "startingTarget" | "done" | "aborted"; /** * What a handover target reports when it has applied: the envelope-level * summary the job's coordinator writes, carried opaque through the worker * and the Manager and read here. Every field optional on the wire; a job * that sent bytes this build cannot read yields `undefined` rather than a * failed handover. * @public */ export interface MigrationApplySummary { /** Participant ids whose apply completed, in the order they completed. */ applied: string[]; /** Participant ids whose apply rejected or ran out of budget, with why. */ failed: { id: string; reason: string; }[]; /** ms between the first participant going live and the last. */ skewMs: number; } /** * Migration state, kept entirely in AutoManager. "No concurrent * migrations per Job" — `sourceJobId` is the primary index; * `migrationId` is a stable opaque handle for ops tooling. * * `sourceRole` is the source's instance key (the slot for a job that * has never migrated, `primary\@xxxxxxxx` after one has); `targetRole` * is the key the target wears (`migrationTargetRole(slot, id)`). * * @public */ export interface MigrationState { migrationId: MigrationId; sourceJobId: JobId; sourceRole: Role; sourceNodeId: NodeId; strategy: MigrationStrategy; /** `migrationTargetRole(slot, migrationId)`, slot = the source's runtime slot. */ targetRole: Role; /** Set once placement has chosen. */ targetNodeId?: NodeId; startedAt: Date; phase: MigrationPhase; phaseHistory: { phase: MigrationPhase; at: Date; }[]; /** Set when phase becomes "aborted"; undefined while in flight or on done. */ abortReason?: string; /** * Opaque, the SOURCE's answer to GetState in `proposing` (handover). * Rides out on the target's StartJob as `migrationInitialState`. * Undefined until the source has answered; an empty array is a * perfectly good answer — a job with nothing to hand over still * accepts, and that is not a refusal. */ initialState?: Uint8Array; /** Opaque, from MigrationTargetReady (handover). Relayed to the source. */ targetPayload?: Uint8Array; /** Opaque, from MigrationSourceStoppedOutput (handover). Relayed to the target. */ sourcePayload?: Uint8Array; /** * The target's own account of its apply, from MigrationTargetApplied * (handover): which participants went live, which did not, and the spread * between the first and the last. Past the point of no return a failure * here is reported, not fatal — a handover is `done` with `failed` * non-empty, and this is the only record of that (task 90, fix-list 41). * Undefined for a target that sent none (a job predating the summary). */ applySummary?: MigrationApplySummary; /** * True from the moment StopJob(source) is sent. Before it, an abort is * a clean rollback (target stopped, source untouched). After it the * saga only fails forward: it keeps waiting on the target, and if the * target is lost the migration is marked aborted and ordinary * recovery re-places the job. */ pastPointOfNoReturn: boolean; } /** * Coarse connection state surfaced by `onConnectionStateChange`. Maps * to gRPC's `ChannelConnectivityState` values via NorskManager's * existing watcher. * * @public */ export type ConnectionState = "idle" | "connecting" | "ready" | "transientFailure" | "shutdown"; /** * Phase C4 — per-policy recovery tunables, keyed by `OnRuntimeFailure`. * * Only the two re-placing policies appear. `drop` is absent by * construction: it never restarts, so a restart ceiling cannot apply to * it. Its old entry carried a lone `onFailure: "drop"` field that * nothing read — a tautology restating its own key — so it went with the * rename, along with the matching `onFailure: "restart"` on the other * two. * * Spare preference is likewise NOT configurable here: it follows from * the policy itself, which the recovery path reads directly. * * Omitted entries fall through to the flat `AutoSettings` fields, then * to compiled-in defaults. * * @public */ export interface FailurePolicyConfig { preferSpare?: { /** Default: 6 */ maxRestartsPerHour?: number; }; avoidSpare?: { /** Default: 6 */ maxRestartsPerHour?: number; }; } /** * Phase C4 — placement / recovery tunables. Same fields as the flat * top-level settings; this typed grouping is the recommended path. * Omitted fields fall through to the flat settings, then to * compiled-in defaults. * * @public */ export interface PlacementConfig { /** Default: 5_000 ms. */ rejectionBackoffMs?: number; /** Default: 3. */ rejectionEscalationCount?: number; /** Default: 600_000 ms. */ unhealthyCooldownMs?: number; /** Default: 30_000 ms. */ failureBackoffMs?: number; /** Default: 120_000 ms. */ jobStartupGraceMs?: number; } /** * Phase C6 — keep `targetCount` warm-running nodes ready per pool. * The reconciler provisions short, terminates * excess. Cluster pools cannot auto-scale — `targetCount` there * means "refuse to place onto a spare-tagged node unless this is a * recovery placement"; reconciler emits `onSpareUnderflow` and leaves * the rest to operator IT. * * @public */ export interface HotSpareConfig { pool: string; targetCount: number; /** The shape to provision spares with (instance type, region, …). */ spec: NodeSpec; /** * Phase G — product-template names this spare should be PRE-WARMED for. * On provision (and whenever the spec changes) the reconciler sends the * node's worker a prefetch of these templates' images, so the spare is * warm before anybody needs it. Warmth itself is still observed, never * declared: `warmFor` only says which pulls to start. * * Independent of `reservedFor`. A spare can be warm for a template it is * not reserved for, and reserved for one it was never asked to pull. */ warmFor?: string[]; /** * Phase G — product-template names this spare is set aside for. With * `exclusive: false` (the default) it is a preference: anyone may take * the spare, these templates just score it first. With `exclusive: true` * it is a guarantee: nobody else may have it. */ reservedFor?: string[]; /** * Phase G — is `reservedFor` a guarantee? Default false. The operator * opts in to exclusivity, because the cost of it is real: a spare nobody * else may take can sit idle while a job waits for a cold node. */ exclusive?: boolean; } /** * Phase G — the editable part of a hot spare's spec: what it is warmed * for, who it is reserved for, and whether that reservation is a * guarantee. Set from a `HotSpareConfig` at provision time and editable * per node at runtime through `updateHotSpareNode`. * * @public */ export interface SpareSpec { warmFor: string[]; reservedFor: string[]; exclusive: boolean; } /** * What `updateHotSpareSpecs` did to the live list, keyed by pool (the * entry's key): entries that were not there before, entries that are gone, * and entries whose target, spec, warmth or reservation differs. * * @public */ export interface HotSpareSpecsDiff { added: string[]; removed: string[]; changed: string[]; } /** * Phase C6 — what kind of node the spare reconciler should * provision. For elastic (aws/oci) pools, `instanceType` is * required. For cluster pools, the spec is only used to constrain * which existing cluster nodes are eligible to be tagged as spare * (future). * * @public */ export interface NodeSpec { /** AwsInstanceType | OciShape; required for elastic pools. */ instanceType?: string; /** Cloud-specific region or AZ; pass through to provider. */ region?: string; az?: string; } /** * How long `drainNode` waits for the jobs it could not migrate off the * node before giving up (`partial`) or, with `force`, terminating the * node with them still on it. Ten minutes: long enough for a handover to * run its course and an operator to notice a stuck job, short enough * that a drain nobody is watching does not hold a cordoned node all day. * @public */ export declare const DEFAULT_DRAIN_TIMEOUT_MS: number; /** * Where a node drain is. In flight: `migrating` (at least one job's * migration is running), `waiting` (no migration in flight, but jobs the * drain could not move are still on the node — until the timeout, or * until they leave by other means), `terminating` (the node is empty * and its terminate has been issued). Terminal: `done` (terminated), * `partial` (the timeout passed with jobs still on the node and no * `force`: nothing was terminated, the node stays cordoned, `jobs` * names the stuck ones), `failed` (the terminate RPC failed, or the * drain was cancelled by `uncordonNode` — `reason` says which). * @public */ export type DrainPhase = "migrating" | "waiting" | "terminating" | "done" | "partial" | "failed"; /** * One job's fate within a drain. `migrating`: its migration is in * flight; `migrated`: it reached the target; `moved`: it left the node * by other means (recovery re-placed it, it was deleted); `stuck`: it * could not be migrated — the saga refused it (not running, not an * AutoManager-placed job), placement found nowhere, or the migration * aborted — and it is still on the node; `terminated`: it was still on * the node when a `force` drain terminated it. * @public */ export type DrainJobStatus = "migrating" | "migrated" | "moved" | "stuck" | "terminated"; /** @public */ export interface DrainJobState { jobId: JobId; status: DrainJobStatus; /** The migration the drain started for this job, once it has one. */ migrationId?: MigrationId; /** Why the job is `stuck`. */ reason?: string; } /** * A node drain's state — the snapshot `drainStatus` returns and * `onDrainProgress` carries. * @public */ export interface DrainState { nodeId: NodeId; phase: DrainPhase; started: Date; timeoutMs: number; force: boolean; jobs: DrainJobState[]; /** Why the drain ended `partial` or `failed`, or (with `force`) what it * terminated over. */ reason?: string; } /** @public */ export interface DrainOptions { /** How long to wait for jobs that cannot be migrated. Default * `DEFAULT_DRAIN_TIMEOUT_MS`. */ timeoutMs?: number; /** At the timeout, terminate the node with its remaining jobs on it * (they recover per their failure policy, as `evictNode` does) rather * than ending `partial`. */ force?: boolean; /** * The migration strategy each job's migration runs with; undefined for * a job means `migrateJob`'s default (`stopThenStart`). AutoManager * knows nothing about product manifests, so a caller that does (norsk-mgr * resolves the declared strategy from the job's template) passes it here. */ strategyFor?: (jobId: JobId) => MigrationStrategy | undefined; } /** * What a provider's notice says it is doing with a node. A * `spot-interruption` is the terminal warning (EC2 gives two minutes); * a `rebalance` is a recommendation that the capacity is at elevated risk * of interruption, with no time attached — it is usually followed by the * interruption notice. * @public */ export type NodeInterruptionKind = "spot-interruption" | "rebalance"; /** @public */ export interface NodeInterruptionNotice { kind: NodeInterruptionKind; /** When the provider reclaims the node. Absent for a rebalance * recommendation, or a notice that carries no time. */ deadline?: Date; /** The migration strategy per job, as `DrainOptions.strategyFor`. */ strategyFor?: DrainOptions["strategyFor"]; } /** * A node the provider is taking back, and the drain running it down — * the snapshot `nodeInterruption` returns and `onNodeInterrupted` carries. * @public */ export interface NodeInterruptionState { nodeId: NodeId; kind: NodeInterruptionKind; deadline?: Date; /** When the (latest) notice was handled. */ noticedAt: Date; drain: DrainState; } /** * How far ahead of the provider's deadline an interruption drain gives * up on migrating and terminates the node with whatever is still on it: * the terminate has to be issued, and the survivors re-placed, before the * host is pulled from under them. Thirty seconds off EC2's two-minute * warning leaves the 90 s below. * @public */ export declare const SPOT_INTERRUPTION_DRAIN_MARGIN_MS = 30000; /** The drain timeout for a spot interruption notice that carries no * deadline: EC2's two-minute warning, less the margin. @public */ export declare const DEFAULT_SPOT_INTERRUPTION_DRAIN_MS: number; /** The drain timeout for a rebalance recommendation, which names no * time: long enough for handovers, short enough to be off the node * before the interruption that usually follows. @public */ export declare const DEFAULT_REBALANCE_DRAIN_MS: number; /** * Per-phase deadlines for in-flight migrations. Set any field to * override its default; omitted fields use the compiled-in default * below. Expiry before the point of no return aborts the migration; * after it (`stoppingSource`, `startingTarget`) the saga fails forward. * * @public */ export interface MigrationConfig { /** Default 30_000 ms — handover: the source has this long to answer the * propose (GetState) before the migration is abandoned. Expiry aborts * with no target ever placed. */ proposeDeadlineMs?: number; /** Default 120_000 ms — the target's first signal after StartJob (running / prepared), every strategy. */ launchingDeadlineMs?: number; /** Default 120_000 ms — handover: workflow has this long after target running to signal ready. */ awaitingTargetReadyDeadlineMs?: number; /** Default 30_000 ms — handover: source workflow has this long to stop output after the cutover signal. */ awaitingSourceStopDeadlineMs?: number; /** Default 10_000 ms — handover: target workflow has this long to apply the handover. */ applyingDeadlineMs?: number; /** Default 30_000 ms — the source's JobStopped after StopJob. Expiry proceeds as if stopped. */ stoppingSourceDeadlineMs?: number; /** Default 60_000 ms — stopThenStart: the target's JobRunning after StartPreparedJob. Expiry fails forward. */ startingTargetDeadlineMs?: number; } /** `placeAndProvision`'s answer: a node now holds the job's claim * (dispatched, or deferred to `bindProvisionedJob`), or nothing does. */ export type PlacementOutcome = { kind: "bound"; nodeId: NodeId; } | { kind: "notPlaced"; reason: string; /** When the caller NAMED a target: why that node in particular was * rejected (its FilterReason), or that it was in no pool at all. * Placement's own reason is the generic `noCapacityInAnyPool`, which * says nothing about a pinned node (task 90, fix-list 17). */ targetReason?: string; }; /** * Phase C4 — result of `validateBundleAdmission`. `ok` means the * bundle passes static validation AND every (replica, jobName) * placeDryRun would succeed at the current moment. The two failure * modes are distinct: `validationErrors` is "the bundle spec itself * is broken" (operator must fix); `noCapacity` is "current cluster * state can't host this" (operator must add capacity or wait). * * @public */ export type AdmissionResult = { kind: "ok"; } | { kind: "validationErrors"; errors: ValidationError[]; } | { kind: "noCapacity"; unplaceable: AdmissionUnplaceable[]; }; /** @public */ export interface AdmissionUnplaceable { replicaIndex: number; jobName: string; reason: PlacementFailureReason; } /** * Phase D4 — operational metrics snapshot. Cumulative counters since * `startedAt`; per-pool gauges are point-in-time at call. Consumers * (telemetry pipelines, ops UIs) poll `AutoManager.metrics()` on * their own cadence. * * @public */ export interface AutoManagerMetrics { /** AutoManager.run() entry time; undefined before runInternal starts. */ startedAt?: Date; /** Milliseconds since startedAt, 0 if not yet started. */ uptimeMs: number; placementsAttempted: number; /** * Placement that successfully dispatched its wire call (startJob / * createAwsNode / createOciNode). Counted post-RPC, so a successful * decision whose dispatch then fails does not bump this counter — * see `placementsDispatchFailed`. */ placementsSucceeded: number; /** Placement engine returned `failure` (no suitable target). */ placementsFailed: number; /** * Decision succeeded but the downstream RPC (startJob / * createAwsNode / createOciNode) rejected. The job is left * unplaced; AutoManager surfaces the error via onError. */ placementsDispatchFailed: number; jobsRejected: number; bundlesBroken: number; /** Jobs dropped rather than restarted (`onRuntimeFailure: drop`, * non-essential past the ceiling, invalid spec). */ jobsDropped: number; pools: PoolMetrics[]; /** Bundles currently marked broken by the restart-rate ceiling. */ brokenBundleIds: BundleId[]; } /** @public */ export interface PoolMetrics { poolName: string; nodeCount: number; /** Spare-tagged, running, no jobs assigned. */ liveSpares: number; /** Sum of `targetCount` across hotSpares entries for this pool. */ spareTarget: number; /** * Mean reservedCapacity / totalCapacity across nodes with inventory. * 0 if there are no nodes with inventory. */ usedCapacityFraction: number; } /** * Payload for `onPlacementDecision`. The `bundleId`/`replicaIndex`/ * `jobName` triple identifies which bundle member the decision was made * for; `result` is the placement engine's verdict; `trace` is the * per-pool filter/score breakdown described in §6 of the design. * * @public */ export interface PlacementDecisionInfo { jobId: JobId; bundleId: BundleId; replicaIndex: number; jobName: string; result: PlacementResult; trace: PlacementTrace; at: Date; } /** * Payload for `onPlacementFailed`. `triedPools` is the ordered list of * tier names AutoManager attempted within the bundle's `pool` before * giving up. * * @public */ export interface PlacementFailedInfo { jobId: JobId; bundleId: BundleId; replicaIndex: number; jobName: string; reason: PlacementFailureReason; triedPools: string[]; at: Date; } /** * Payload for `onResilienceDegraded`. Fired when a bundle's backup * (replicaIndex 1) is placed inside a failure domain its resilience policy * asked to avoid, because no compliant node had capacity. `violated` lists * the anti-affinity flags that couldn't be honoured. The placement still * happened — a degraded backup beats no backup — but DR posture is reduced. * * @public */ export interface ResilienceDegradedInfo { jobId: JobId; bundleId: BundleId; replicaIndex: number; jobName: string; violated: ResilienceFlag[]; /** Set when the primary is on interruptible (spot) capacity with no * durable backup — see ResilienceDegradation.interruptiblePrimary. */ interruptiblePrimary?: boolean; at: Date; } /** * Payload for `onJobRejected`. `inventory` is the typed snapshot the * worker sent alongside the rejection so consumers can render "why" in * the same UI tick as the rejection itself. * * @public */ export interface JobRejectedInfo { jobId: JobId; nodeId: NodeId; reason: ManagerPB.JobRejected["reason"]; inventory?: TypedNodeInventory; at: Date; } export declare const MIN_RECOVERY_INTERVAL_MS = 5000; export declare const CLOSE_ABORT_GRACE_MS = 5000; /** * How long, after a (re)connect, a job stays bound to a node its history * names but the daemon has not (yet) reported. A Manager daemon restart * sends its initial state as soon as it is up, while the EC2 rediscovery * of the nodes it had — one hello every few seconds — is still running; * until this, a job whose node had not re-hello'd read as unplaced, was * re-placed onto whatever node HAD, and when its real node then hello'd * carrying the same job the daemon terminated that node as a duplicate. * Observed on norsk-e2e 2026-09-14 16:53Z: two productions and two nodes * lost to one Manager restart. Five minutes covers a slow rediscovery; * a node that never comes back is released for re-placement after it. */ export declare const HYDRATION_NODE_GRACE_MS: number; export { PREFETCH_RETRY_BACKOFF_MS, SPARE_PROVISION_RETRY_BACKOFF_MS } from "./hotSpares"; export declare class AutoManager { norsk: NorskManager; settings: AutoSettings; clock: Clock; jobMap: Map; private pendingProvisions; private pendingProvisionTimers; nodeMap: Map; bundleMap: Map; inventoryMap: Map; private rejectionsPerNode; private unhealthyNodes; private recentFailures; private restartHistory; private readonly lastCountedDispatchRefusal; private inventoryContractWarned; private pendingPlacementTimers; private pendingStopTimers; private pendingPlacementRetries; private brokenBundles; private startupGraceTimers; private unseenBoundNodes; private hotSpares; private hotSpareReconcileInterval?; private cordonedNodes; /** Nodes `onNodeTerminated` has fired for — see `issueTerminate`. */ private terminateSignalled; private drains; private reclaimedNodes; private metricsCounters; private migrationSaga; private nodeServiceReconciler; private nodeServiceReconcileInterval?; /** * The moment `runInternal()` first entered (i.e. the eventStream was * established). Undefined until then. Powers "AutoManager up for N * minutes" affordances. * @public */ startedAt?: Date; private cleanupInterval?; private closed; private hydrated; /** @public */ static run(settings: AutoSettings): Promise<[AutoManager, Promise]>; /** @public */ terminateNode(nodeId: NodeId): Promise; /** * Every terminate AutoManager itself issues goes through here, so * `onNodeTerminated` fires exactly once per node whichever path (and * however many retries of it — the orphan-spare reaper re-issues on * every tick until the node stops) got there. The signal fires at issue * time; callers keep their own handling of the RPC's outcome. (No await * before the signal: the body runs synchronously up to the return, so the * signal still fires at issue time, before any caller's await resumes.) */ private issueTerminate; /** * Phase C8 — cordon a node. AutoManager projects this onto every * placement decision: cordoned nodes are filtered out, so no new * placements land on them. Existing jobs continue running. * Idempotent. * @public */ cordonNode(nodeId: NodeId): void; /** * Phase C8 — un-cordon a node, restoring it to the placement pool. * Idempotent. * @public */ uncordonNode(nodeId: NodeId): void; /** * Phase C8 — whether a node is currently cordoned by AutoManager. * @public */ isCordoned(nodeId: NodeId): boolean; /** * Evict a node: cordon it (so no new placements land here) and * terminate it at once — Manager emits `nodeStopping` for every job * hosted there, and the policy-aware recovery path re-places them onto * other nodes (excluding this cordoned one), with a gap. `drop` jobs * are dropped; `preferSpare` prefers hot spares. This is what * `drainNode` did until 2026-09-16; `drainNode` now moves the jobs * first, and this is the blunt instrument for when that is not wanted. * @public */ evictNode(nodeId: NodeId): Promise; /** * Drain a node: cordon it, migrate every job off it, and terminate it * once it is empty. * * Each job bound to the node is started on a migration through the * same saga `migrateJob` runs (placement elsewhere — the source node is * never a candidate, and it is cordoned besides; a new node may be * provisioned in an elastic pool). A job the saga refuses (not running, * not AutoManager-placed), one placement finds nowhere for, or one * whose migration later aborts is `stuck`: it stays running where it is * until the timeout (`DEFAULT_DRAIN_TIMEOUT_MS` unless given), and * then either the drain ends `partial` — nothing terminated, the node * still cordoned, `jobs` naming the stuck ones — or, with `force`, the * node is terminated with them on it and they recover per their * failure policy, as `evictNode` would have had them. A migration that * fails forward past its point of no return is re-placed by ordinary * recovery, which counts as the job leaving the node. The terminate goes * through `issueTerminate`, so `onNodeTerminated` fires exactly once. * * Resolves as soon as the migrations have been started (their placement * and dispatch, or a handover's propose, settle afterwards) with the * drain's state; progress is pushed through `onDrainProgress` and read * back through `drainStatus`. One drain per node: a second call while * one is in flight returns that one's state. `uncordonNode` cancels a * drain that has not yet issued its terminate. * @public */ drainNode(nodeId: NodeId, opts?: DrainOptions): Promise; /** * The drain in flight for a node, or the last one that ran; undefined * for a node never drained by this instance. * @public */ drainStatus(nodeId: NodeId): DrainState | undefined; /** * A provider has said it is taking a node back — a spot interruption * notice (EC2: two minutes' warning) or a rebalance recommendation (no * time given). Cordon the node and run it down with a FORCED drain * (`drainNode`) whose timeout fits the deadline: `deadline − now − * SPOT_INTERRUPTION_DRAIN_MARGIN_MS`, or with no deadline the kind's * default (`DEFAULT_SPOT_INTERRUPTION_DRAIN_MS`, * `DEFAULT_REBALANCE_DRAIN_MS`). Jobs that migrate in time land * elsewhere; whatever is still on the node at the timeout goes down with * it and is recovered per its failure policy — and none of it counts * against the restart ceiling: the node was reclaimed, the jobs did not * fail (`recoverLostNode` reads `reclaimedNodes`). * * A notice for a node already draining (the interruption that follows a * rebalance, say) keeps that drain — its migrations carry on — and * tightens its timeout when the new one ends sooner. `uncordonNode` * withdraws the notice along with the drain. Idempotent per notice; * `onNodeInterrupted` fires once per call. * * Who calls this: the event loop, on the daemon's `nodeInterruptionNotice` * (the worker on the instance watches IMDS — spot/instance-action and the * rebalance recommendation — and the daemon relays it); and norsk-mgr, * from any other source it has for a notice (an operator hook). * @public */ handleNodeInterruption(nodeId: NodeId, notice: NodeInterruptionNotice): Promise; /** * The interruption notice in force for a node, with its drain; undefined * for a node no provider has reclaimed (or one that has since stopped). * @public */ nodeInterruption(nodeId: NodeId): NodeInterruptionState | undefined; /** A drain in flight gets a new, shorter time to go: it becomes forced * (a reclaim leaves no `partial` to end in) and its timer is re-armed * when `remainingMs` ends before the current timeout would. */ private tightenDrain; /** Jobs whose placement of record is this node (`bindings`). A * migration TARGET bound here is not yet on the node's books. */ private jobIdsBoundTo; /** Start one job's migration for a drain and track it. The saga records * its state before its first await, so the id is known on return; a * refusal before that shows up as the promise's rejection. */ private startDrainMigration; /** * Bring a drain's state up to date with where its jobs are. Called * after every stream event and whenever a migration ends: jobs that * have left the node are marked so, jobs that turned up on it are * started on migrations of their own, and an empty node is * terminated. */ private reconcileDrain; /** Every drain, after a stream event that may have moved a job. */ private reconcileDrains; /** The node is empty: issue its terminate (once, whichever path — see * `issueTerminate`) and end the drain on the RPC's outcome. */ private terminateDrainedNode; private drainTimedOut; /** End a drain that has not issued its terminate. No-op otherwise. */ private cancelDrain; /** A migration the drain started has ended: an abort leaves its job * stuck if it is still on the node (a fail-forward re-place moves it, * which the reconcile sees). */ private noteDrainMigrationEnded; /** * Shut down this AutoManager. Cancels every timer this instance owns, * stops the event loop, aborts every in-flight migration, and closes * the underlying NorskManager — which resolves the `Promise` * returned from `run()`. Idempotent. * * An instance is never reconnected: norsk-mgr disposes it with * `close()` and runs a fresh one, so this is the only place an * in-flight migration can be wound down. Before the point of no * return that is the clean rollback (abort to both keys — a source * that answered a handover propose is locked until it hears it — * and StopJob for the target, which wears a key nobody will address * again); the source keeps running. Past it there is no rollback: * the source is stopping or stopped, so the migration is only marked * lost and the next instance rebuilds the job's placement from * history. A `migrateJob` still parked on its propose rejects. * * The channel stays open just long enough for those aborts to be * sent (`CLOSE_ABORT_GRACE_MS` at most); with nothing in flight it * closes immediately. * @public */ close(): void; /** * Alias for `close()`. Provided for symmetry with the * `using`-style disposal conventions emerging in TypeScript. * @public */ dispose(): void; /** * AutoManager v1 — submit a new bundle. The bundle is validated by * Manager (per §4.3) and expanded into N replica jobs server-side; * AutoManager mirrors the resulting state via the bundleUpdated + * jobUpdated events. The returned bundleId matches the one Manager * assigns (typically the same as bundle.bundleId, but Manager has * the final word). * * @public */ createBundle(bundle: TypedBundle): Promise; /** * AutoManager v1 — update an existing bundle. Always sends * `state: live`: the state field is a daemon-owned verdict * (markJobFailed sets FAILED, the delete saga DELETING) that * read-modify-write callers naturally echo back — norsk-mgr's Retry * sent back the failed bundle it had just read, and the daemon * persisted FAILED straight over the revived production, leaving a * healthy re-expanded bundle labelled Failed forever (observed live * 2026-08-26). An operator pushing a spec update is reviving the * bundle by definition. * @public */ updateBundle(bundle: TypedBundle): Promise; /** * AutoManager v1 — delete a bundle and all its replica jobs. * @public */ deleteBundle(bundleId: BundleId): Promise; /** * AutoManager v1 — activate a dormant (start_mode = "manual") bundle * job. Manager removes the dormant tag and re-emits jobUpdated; * AutoManager's placement path picks the job up via the standard * flow. See §10.2 of the design. * @public */ startBundleJob(bundleId: BundleId, replicaIndex: number, jobName: string): Promise; /** * AutoManager v1 — deactivate a bundle job (tag-flip). * * Runtime termination of an in-flight workload and empty-node * release with linger are pre-go-live work tracked as Phase E.5 * (§10.2.1). Until they land, calling stopBundleJob on an active * job marks it dormant for future placement but does not stop the * currently-running worker. * @public */ stopBundleJob(bundleId: BundleId, replicaIndex: number, jobName: string): Promise; /** * AutoManager v1 — typed view of the jobs Manager has told us about. * `JobWithHistory` carries the state-transition log for each job so * consumers can render lifecycle timelines without a separate * round-trip. * @public */ jobs(): JobWithHistory[]; nodes(): NodeSummary[]; /** * AutoManager v1 — typed view of the bundle definitions Manager has * told us about. Mirror only; AutoManager owns no durable state. * @public */ bundles(): TypedBundle[]; /** * Phase D4 — snapshot of AutoManager's operational metrics. * Cumulative counters since `startedAt`; per-pool gauges computed * at call time. Cheap to call repeatedly — consumers poll on their * own cadence and compute deltas if they want rates. * @public */ metrics(): AutoManagerMetrics; private computePoolMetrics; /** * Look up a single bundle by id. Convenience over `bundles().find(...)`. * @public */ bundleById(bundleId: BundleId): TypedBundle | undefined; /** * Every job (with history) that belongs to a given bundle. Derived * from `tags.bundleId` on the job entries in jobMap. Returns an empty * array if the bundle is unknown or has no jobs. * @public */ jobsForBundle(bundleId: BundleId): JobWithHistory[]; /** * The node the given job is currently placed on, or undefined if * the job hasn't been placed (or has been deleted). Useful for * operator UIs that render "job X is on node Y" and for proxy * routers that need to know which worker hosts each job. * * O(1) lookup against the job's binding — set at placement * time, deleted on jobDeleted, rebuilt from job history on R7 * reconnect. Works regardless of whether the node hosts one job * (cloud, today) or many (cluster, or any future cloud N:1) * because the assignment is recorded against the JOB rather than * scanned out of NODE tags. * @public */ nodeForJob(jobId: JobId): NodeSummary | undefined; /** * Current placement of every job AutoManager has bound to a node, * newest-first by decision timestamp. Mirrors `nodeForJob` but * returns the full `PlacementDecisionInfo` (so consumers get the * pool, trace, replicaIndex etc., not just the node) — and exposes * it as queryable live state rather than only via the * `onPlacementDecision` event stream. * * Hydrated jobs (placed in a previous AutoManager generation, picked * up on R7 reconnect) get an entry with an empty `trace.attempts` * and a decision inferred from job history. Consumers that *need* * a real trace should subscribe to `onPlacementDecision` for live * decisions; this getter is for "where is everything right now?" * operator views. * * @public */ placements(): PlacementDecisionInfo[]; /** * O(1) lookup for the most recent placement decision binding `jobId` * to a node. Returns undefined for jobs that have never been placed * (or whose placement attempts have only failed). @public */ placementForJob(jobId: JobId): PlacementDecisionInfo | undefined; /** * Run the placement engine against AutoManager's current inventory * snapshot and configured pools, without actually placing or * provisioning anything. Powers preview affordances in operator * UIs ("if I submitted this bundle now, where would each replica * land?"). * * `jobName` is optional; if omitted the *first* job in the bundle * is used. For multi-job bundles, supply the name to get a decision * specific to that job. * * Returns undefined if the bundle's named job is missing or the * replica index is out of range. * @public */ placeDryRun(bundle: TypedBundle, replicaIndex: number, jobName?: string): { result: PlacementResult; trace: PlacementTrace; } | undefined; /** * Static validation of a bundle against AutoManager's currently * configured pools. Equivalent to calling the module-level * `validateBundle` with `PoolDescriptor[]` derived from * `AutoSettings.placementPools` — elastic pools project their * `candidateInstanceTypes` into node profiles; cluster pools project * their currently-known live inventory. * @public */ validateBundle(bundle: TypedBundle): ValidationResult; /** * Phase C4 — packaged admission check. Combines `validateBundle` * (static spec sanity) with `placeDryRun` for every (replica, * jobName) pair. Returns `ok` only when both pass for every * member; otherwise reports the structured failure. Use as a * pre-submit guard in operator-facing tooling. * * Every policy rejects at admission on "no capacity right now" — * there's no per-policy differentiation here. (Re-placing policies * get *queued* internally on capacity-after- * submit failures via the rejection re-place path, but that's a * runtime concern, not admission.) * * @public */ validateBundleAdmission(bundle: TypedBundle): AdmissionResult; private poolDescriptorsForValidation; /** * AutoManager v1 — typed inventory snapshot for a single node, or * undefined if the worker hasn't (yet) advertised one. * * Reservations are the same derived overlay placement scores against * (see `inventoriesAsNodeViews`): the wire values arrive as 0 / [] — * workers advertise raw hardware — so without the overlay a consumer * (e.g. norsk-mgr's /api/inventory → the Nodes page) shows a node * hosting jobs as "0/64 reserved" while placement correctly sees it * as loaded. One view of the truth, both here and there. * @public */ inventory(nodeId: NodeId): TypedNodeInventory | undefined; /** * AutoManager v1 — typed inventory snapshots for every node we know * about. Iteration order is insertion order. Reservations carry the * same derived overlay as `inventory()` above. * @public */ inventories(): { nodeId: NodeId; inventory: TypedNodeInventory; }[]; /** * The one place a wire inventory becomes a typed one. The converter * enforces the `utilizationCap` contract (a fraction of totalCapacity; * an absolute — the vCPU count a pre-fix worker sent — is read as 1 so * it cannot inflate the node) and reports the violation; this reports it * to the operator ONCE per node rather than on every poll. */ private typedInventoryFrom; private handleNodeInventoryUpdated; private handleJobRejected; /** * The ending for a job whose spec the worker refused as unrunnable * (`invalidSpec`): release the provision-time binding the daemon has * already dropped, mark the job terminally failed on the wire, and end * the production — the same ending the restart ceiling gives, reached * on the first rejection because a spec fault cannot improve with * retries. A migration target rejected this way is the saga's: the * source still runs, and aborting the migration is the right ending. */ private endJobForInvalidSpec; private recordRejection; /** Clear every placement-exclusion mark for a node that has demonstrably * recovered (a job just reached `running` on it). Fires `onNodeHealthy` * so consumers mirroring the unhealthy set (e.g. norsk-mgr's state * counter) can drop it too. */ private clearNodeHealthMarks; private currentExcludeSet; /** * Phase C2 — restart-rate check. Records the current attempt; if the * trailing-hour count crosses the job's `maxRestartsPerHour`, marks * the bundle broken, fires `onBundleBroken`, and returns true * (caller must short-circuit). Returns false when within budget. * * WHAT THE CEILING MEASURES — three rules, all of them about not * counting the same trouble twice: * * - It counts FAILURES, not attempts. A recovery re-entering after * its backoff (`retryOfCountedFailure`) is the same failure still * being worked on, and is not counted again. * - A run of consecutive dispatch refusals for the same job, with the * same reason, against the same node is ONE condition, not N * crashes (`countsTowardCeiling: false` — see * `repeatOfCountedCondition`). The daemon holding a leaked binding * and answering `Job already running on node` every 5 s used to * burn a ceiling of 100 in half an hour on a healthy production * (2026-09-09). A DIFFERENT reason, or a different node, is a new * count; so is the same refusal after the job has run in between. * - It counts nothing a provider RECLAIM does. A node lost to a spot * interruption (`handleNodeInterruption`) takes its jobs with it * through no fault of theirs; their re-place is recorded for the * backoff and not counted (`reclaimed` on the recovery arg). * - It counts nothing a MIGRATION does. A handover is a planned move, * not a restart: the target is placed through the `migration` arm of * `placeAndProvision`, which is neither initial nor recovery, and * the source's `JobStopped(migration)` is consumed by the saga. The * one exception is deliberate — a fail-forward past the point of no * return has genuinely lost the job, and its re-place counts once, * like any other loss. * * A non-counting attempt is still RECORDED, because the minimum-interval * backoff paces off the newest entry — skipping the record would let a * repeat storm at wire speed. */ private shouldBreakOnRestart; /** * How many failures the restart ceiling currently attributes to this * job — the number `maxRestartsPerHour` is compared against, within * the trailing hour. Attempts the ceiling deliberately does not count * (a backoff retry of a failure already counted, a repeat of a standing * dispatch refusal) are excluded, and a migration contributes nothing; * see `shouldBreakOnRestart` for what that means. * @public */ restartCount(jobId: JobId): number; /** * Whether the named bundle has tripped the restart ceiling and is * being held by AutoManager. Cleared by `bundleUpdated` (operator * pushed a fresh spec) or `bundleDeleted`. * @public */ isBundleBroken(bundleId: BundleId): boolean; /** * Migrate a job's running instance to another node. `strategy` picks * the saga (`stopThenStart` by default, `overlap`, or `handover`); * `targetNodeId` pins the target; `sourceRole` names the instance when * it is not the job's current one; `overrideReservation` lets the * target take an exclusively reserved hot spare. Resolves with the * migration id once the target is placed and launching; rejects — with * the migration aborted — when the source refuses, nothing can host the * target, or the target's dispatch fails. * @public */ migrateJob(sourceJobId: JobId, opts?: { targetNodeId?: NodeId; strategy?: MigrationStrategy; sourceRole?: Role; overrideReservation?: boolean; }): Promise; /** * Abort an in-flight migration cleanly: MigrationAbort to both keys, * StopJob for the target, its reservation released, the source * untouched. No-op (with a debug log) for an unknown migration id or * one already done/aborted. Rejects once the migration is past the * point of no return — the source has already been told to stop, so * there is nothing to roll back to; the saga will complete into the * target, or fall into ordinary recovery if the target is lost. * @public */ abortMigration(migrationId: MigrationId, reason: string): Promise; /** All known migrations (active + done + aborted). @public */ migrations(): MigrationState[]; /** Look up a single migration by id. @public */ migrationById(migrationId: MigrationId): MigrationState | undefined; /** Migration active for the given source jobId, if any. @public */ migrationForJob(jobId: JobId): MigrationState | undefined; /** * Forget a TERMINAL migration — `migrations()` keeps every migration * this AutoManager has ever run, which is an unbounded list an operator * has no other way to shorten. Refuses (throws) an in-flight one: abort * it first. Returns false for an id that is not (or is no longer) * known, so a double-clear is a no-op rather than an error. * @public */ forgetMigration(migrationId: MigrationId): boolean; /** * Forget every terminal migration, leaving the in-flight ones alone. * Returns the ids forgotten. @public */ forgetTerminalMigrations(): MigrationId[]; private inFlightMigrations; /** * The instance KEY of the job's current instance: the key its last * completed migration left it under, else its slot. This is the role * every RPC that addresses the RUNNING instance must use. */ private roleForJob; /** The job's runtime SLOT ("primary" / "backup"), from its replicaIndex tag. */ private slotForJob; private cfgRejectionBackoffMs; private cfgRejectionEscalationCount; private cfgUnhealthyCooldownMs; private cfgFailureBackoffMs; private cfgJobStartupGraceMs; /** The ceiling for one job: its own override first, then the per-policy * setting, then the flat one, then the compiled-in default. * * The per-job override leads because it is the most specific statement * anyone made — an operator who says "this encoder gets 2 tries an hour" * means it regardless of what the deployment-wide default happens to be. */ private cfgMaxRestartsPerHour; private placeAndProvision; /** Does any node other than `except` advertise the named capability at * all (reserved or not)? Used to word a placement refusal. */ private anyOtherNodeAdvertises; private inventoriesAsNodeViews; /** Drop a pending provision (dispatch failure / node death / timeout): * the prospective view disappears, every job bound to the never-born * node is unbound, and the reconcile sweep re-places them — elsewhere, * or via a fresh provision. */ private abandonPendingProvision; /** Rebuild a pending-provision entry for a still-booting node found at * hydration time: shape from the instanceType tag (providerMetadata * fallback for AWS nodes predating the tag) resolved against the * CURRENT settings' machine-type catalogue. The timeout re-arms fresh * from reconnect rather than remaining-from-createdAt — conservative * against clock skew across generations; a truly dead provision costs * one extra window before abandonment re-places its jobs. Without a * resolvable shape there is no prospective view to synthesize — the * node still binds its job on arrival, but can't be joined meanwhile. */ private hydratePendingProvision; /** Timeout fire: the node never reported. Same abandonment as a create * failure — jobs must not wait on a node that is no longer coming. */ private expirePendingProvision; /** Map a placed jobId to the RunningJob view the placement engine uses * for affinity / spread / exclusivity, via the job's bundle-context * tags. Empty when the job lacks bundle tags. `exclusive` comes from * the occupant's bundle spec so the node-hosts-exclusive filter can * see it without a second lookup inside the pure engine. */ private runningJobFor; /** Sum the resource requirements of the given placed jobs into a node * reservation. Capacity / cores / capabilities come from each job's * bundle spec — the canonical typed requirements the placement engine * scores against, looked up via the job's bundleId / jobName tags. Jobs * whose bundle or spec we don't have yet contribute nothing. */ private derivedReservation; /** * The GPU device a bound job holds, from its placement of record — * `PlacementResult.gpuIndex` on both `place` and `provision` decisions, * which is where placement records the device it chose. A migration * target's claim is in the parked target decision. A job hydrated from * history has no live decision (the synthesised record carries no * device) and so holds no device until it is next placed — the wire * carries none either (no worker populates `assignedJobs`). */ private gpuIndexOfRecord; /** Invert the bindings once: nodeId → the jobs AutoManager placed * there. Reads BOTH the jobs' own bindings and the migration targets, * so a migrating job is counted on its source AND its target for as * long as the overlap lasts — the honest reservation (a full cluster * cannot migrate; hot spares are the answer, not a looser check). */ private placedJobsByNode; /** * Overlay the derived reservation (from the jobs AutoManager placed on * the node) onto a wire inventory, deferring to a non-zero wire value — * if a producer ever advertises real reservations they're authoritative. */ private withReservationOverlay; /** The canonical typed bundle spec for a placed job, looked up via its * bundleId / jobName tags. Undefined for un-managed jobs. */ private specForPlacedJob; /** The canonical typed requirements for a placed job, from its bundle * spec (the same source `placeAndProvision` scores against). */ private requirementsForPlacedJob; /** * Phase C5 — is the named node currently tagged as a hot spare? * Spares are warm-running nodes provisioned (by the C6 reconciler) * with `NodeMetadata.tags["spare"] = "true"`. * * There is deliberately no per-spare dedication. A `forPolicy` field and a * matching `spareBand` node tag used to exist, but placement never read * them — they only partitioned the reconciler's own accounting — so a spare * "reserved" for one policy was reserved from nobody. Which work prefers or * avoids a spare is decided on the JOB, by its onRuntimeFailure. * @public */ isHotSpareNode(nodeId: NodeId): boolean; /** * Register (or replace) the container images a product template needs. * Overlays `settings.templateImages`; the embedder that reads the * template manifest calls this when a template is added or rebuilt. * @public */ setTemplateImages(templateName: string, images: string[]): void; /** * The images a product template needs — runtime registration first, then * settings. Unknown ⇒ empty, which means "no warmth signal" rather than * "needs nothing": `warmth` scores 0 for every node, so placement is * unbiased rather than wrong. * @public */ templateImages(templateName: string): string[]; /** * A spare's live spec, read from its node tags: what it was provisioned * with, as edited since through `updateHotSpareNode`. The tags are the * only copy — the daemon persists an edit and re-applies it when it * rediscovers the node, so the spec survives a reconnect and a restart * on either side (task 90 fix-list 46: an in-memory override here was * lost on every norsk-mgr restart). A node that is not a spare has no * spec. * @public */ spareSpec(nodeId: NodeId): SpareSpec | undefined; /** * Edit a running spare's spec. Adding to `warmFor` starts the pulls * (Steve's case: two spares warmed for `xyz`, a migration of `abc` * needed, so `abc` is added and the pull happens while the migration is * still proposing). Removing from it does NOT delete images — it only * stops us claiming the node is kept warm for that template; the images * stay until the node's own eviction reclaims them. * * Omitted fields keep their current value. Throws if the node is not a * hot spare. * @public */ updateHotSpareNode(nodeId: NodeId, edit: { warmFor?: string[]; reservedFor?: string[]; exclusive?: boolean; }): Promise; /** * Replace the live `hotSpares` list without a reconnect. The embedder * used to reload settings for this, which tore the AutoManager down and * — since `close()` aborts every in-flight migration — killed whatever * was migrating (the IBC-demo landmine). Now: the list is validated as * it would be at boot (the `hotSpares` slice of `validateAutoSettings`, * plus one pool per entry, since the pool is the key) and rejected whole * with the codes; otherwise it becomes the live list and the reconciler * converges on it on its next tick, which runs at once. A removed entry's * unconsumed spares are reaped as orphans; an added one is provisioned; a * lowered target terminates the excess coldest first; a changed * `warmFor` / `reservedFor` / `exclusive` is written to the live spares' * tags (only the fields that changed, so a per-node edit of the others * survives), and the pulls follow. Nothing else is touched — a migration * in flight stays in flight. * @public */ updateHotSpareSpecs(specs: HotSpareConfig[]): Promise; /** * Is this a LOCAL SERVER that is a hot spare (marked through * `setLocalSpare`, not provisioned)? A spare to placement; nobody's to * the reconciler. See `HotSpares.isLocalSpare`. * @public */ isLocalSpare(nodeId: NodeId): boolean; /** * Mark a local server as a hot spare (`spare: true`, optionally with its * `warmFor` / `reservedFor` / `exclusive`), or clear it. The daemon owns * the tag from there and drops it when a job binds, as for an elastic * spare. Rejects for a node that is not a local server or that holds a * job. See `HotSpares.setLocalSpare`. * @public */ setLocalSpare(nodeId: NodeId, edit: { spare: boolean; warmFor?: string[]; reservedFor?: string[]; exclusive?: boolean; }): Promise; /** * Override how long this node lingers idle before the daemon expires it * (fix-list item 8: the node a handover vacated was destroyed before you * could migrate back onto it). `0` restores the global `emptyNodeLinger`. * @public */ setNodeIdleExpiry(nodeId: NodeId, ms: number): Promise; /** * Pull the images of some product templates onto ANY node, spare or not * (task 90, fix-list 39). * * A spare's `warmFor` edit does this and also records the declaration; * an ordinary node — the one a handover just vacated, which is the fastest * target the next handover could have — has no spec to edit, but the pull * is the same pull, and warmth is OBSERVED (`heldImages`) so placement * scores it the moment the images land. Nothing is declared or reserved. * Resolves with the image refs asked for; empty when no template resolves * to any image (no warmth signal exists for it yet), which is not an * error. Rejects for a node this AutoManager does not know. * @public */ prefetchTemplates(nodeId: NodeId, templateNames: string[]): Promise; private handleImagesPrefetched; private reconcileHotSpares; /** * @public * The configured NodeServices. */ nodeServices(): NodeService[]; /** * @public * Nodes on which the named NodeService is currently placed, * keyed by the runtime job's jobId. */ nodeServicePlacementsFor(serviceId: ServiceId): Map; private reconcileNodeServices; private tagsForJob; private poolNameForNode; private tierNameForNode; private azForNode; private cloudForNode; private handleBundleUpdated; private handleBundleDeleted; private clearRestartHistoryForBundle; private nodesByJob; /** jobId → timestamp (ms) of the runtime failure we have already recovered * from, so redelivered `jobUpdated` events for the same failure are not * counted again. Cleared when the job is deleted. */ private readonly handledRuntimeFailures; /** Jobs AutoManager has ruled terminally failed (a policy drop, a ceiling * trip) and told the daemon so with `markJobFailed`. The verdict is on * the wire, not yet echoed: until the job comes back with * `terminalState`, it still reads `active` with no binding, which is * exactly what `wantsPlacementNow` places. The daemon's own * `processJobFailed` records `stopping` AND `stopped`, one `jobUpdated` * each — so the second event of the very failure we just dropped on * used to re-place the job as a fresh initial placement (a new node for * a job the operator has been told is gone). Cleared when the echo lands * (the durable flag takes over), when the job is deleted, and when the * operator relaunches the bundle. */ private readonly terminalVerdictsIssued; private issueTerminalFailure; private handleJobChanged; /** * Re-place a runtime the worker reported as failed, honouring the same * policy as any other recovery (drop / preferSpare / avoidSpare) and * counting against the same restart-rate ceiling — `placeAndProvision` * owns all of that; this only has to unbind and hand over. * * Idempotent by construction: the unbind is the trigger, so a repeated * `stopping/jobFailed` for a job we have already released finds no * binding and does nothing. Without that guard a redelivered event would * count twice against the ceiling and trip it early. */ private recoverFailedRuntime; /** Read the `__stopDateTime` tag off the job, and (re)schedule the * bundle's stop timer accordingly. Tag is set per-job but applies * to the whole bundle — first-job-wins for read; all jobs in a * bundle share the same value at submit. */ private reconcileStopTimer; private scheduleStopTimer; /** * Update (or clear) a bundle's auto-stop time. Calls `updateBundle` * so the new value persists via the daemon's tag store and any other * clients see it through their event streams. Pass `undefined` to * cancel a previously-set stop time. * * Throws if the bundle isn't known to this AutoManager (no live * bundleMap entry); operators wanting to set stop on a not-yet- * streamed bundle should retry after JobPending fires. * @public */ setBundleStopTime(bundleId: BundleId, stopDateTime?: Date): Promise; /** * Snapshot of all currently-scheduled bundle stop timers. UIs render * this alongside `pendingPlacements()` for an "upcoming events" * view. * @public */ bundleStopTimes(): Array<{ bundleId: BundleId; stopAt: Date; }>; /** A job that should be running but currently has no node bound to * it: it carries placement `requirements`, isn't manual-start, is in * a placeable state (`pre` / `active`), and `nodesByJob` is empty. * This is the single source of truth for "this job wants a placement * and doesn't have one" — shared by initial hydration, the per-job * `handleJobChanged` path, and the capacity-driven reconcile sweep, * so the decision lives in exactly one place. */ private wantsPlacementNow; /** * Is this job bound to a node the daemon has yet to report, inside the * grace a (re)connect grants such nodes? Such a job is placed, as far as * anyone can tell, and must not be placed again. */ private boundToUnseenNodeInGrace; /** * After hydration: every node a job is bound to (from its history) that * the initial state did not carry gets a grace. When the node reports in * (`nodeStarted`) the grace is cancelled; when it expires the bindings to * it are released and the jobs re-placed, exactly as if the node had * just been reported stopped. */ private grantHydrationGrace; /** * Re-run placement for every job that wants a node but hasn't got * one. Called whenever a *capacity-increasing* event arrives — a node * comes online (`nodeStarted`), a worker reports fresh inventory * (`nodeInventoryUpdated`, which also covers an un-cordon or a node * that has just freed reserved capacity), or a job finishes and * releases its node (`jobDeleted`). Those are the only transitions * that can turn a previously-unplaceable job placeable, so they are * exactly when re-running the engine is worthwhile — there is no * polling retry, the trigger is always a concrete event. * * Placement stays implemented once: each candidate routes through * `placeOrDefer` (the same entry point hydration and `handleJobChanged` * use), inheriting its scheduling, the in-flight gate, and the * failure path. Jobs already waiting on a schedule timer or a recovery * backoff are skipped — their own timers own them — so a capacity * event never disturbs a future-scheduled or backing-off job. * * Cheap and idempotent (mirrors `reconcileHotSpares` / * `reconcileNodeServices`); safe to call on every relevant event. */ private reconcilePendingPlacements; /** Decide whether to place immediately or defer until the job's * scheduled `startDateTime` (minus pool lead time) arrives. * Re-entry from job-update events: cancels and re-evaluates so a * schedule change moves the timer. Hydrated jobs with `startTime - * lead` in the past place immediately (operator intent was to run; * schedule was a "when", not a "drop if missed"). */ private placeOrDefer; /** Maximum `placementLeadMs` across the bundle pool's tiers (worst case * — guarantees the job is ready by `startDateTime` regardless of which * tier the placement engine ends up picking). Falls back to the per-kind * default when a tier doesn't specify one. Returns 0 if the bundle or * pool isn't found — we'd rather race the schedule than indefinitely * delay placement. */ private placementLeadMsForJob; /** * Currently-scheduled placements (jobs AutoManager has deferred * because their `startDateTime - leadTime` is still in the future). * Returns one entry per pending timer; the UI can render an * "upcoming jobs" list and show when each will go live. * @public */ pendingPlacements(): Array<{ jobId: JobId; scheduledFor: Date; }>; private handleJobDeleted; /** Nodes already dispatched a single-use termination — dedupes the * per-member triggers when a co-located group ends together. */ private singleUseTerminated; /** * A single-use job on `nodeId` has ended. Terminate the node NOW — * never linger for reuse (the flag exists so one event's data can't * leak onto a later event's runtime) — provided: * - the node is provisioned cloud capacity (aws/oci tier). Registered * cluster nodes are operator hardware; terminating them is the exact * provider-mistake the old terminate-on-jobDeleted was removed for. * - no OTHER live job is still bound to it (last-man-out for a * co-located group; validation keeps singleUse uniform across a * group so the last member out is always single-use). */ private maybeTerminateSingleUseNode; private handleNodeStarting; /** Phase C3. Schedule the grace timer; replace any prior timer. */ private armStartupGrace; private cancelStartupGrace; private handleNodeStarted; /** Dispatch startJob for the job a freshly-running node was provisioned * for. Guards make this idempotent and narrow: * - only nodes the AutoManager provisioned (`auto-manager` tag) with a * `jobId` tag (spares carry no jobId); * - only when the job is bound to this node, or to nothing (hydration * after a restart mid-boot: history had no node entry, so the binding is * empty — the tagged node claims the job rather than letting the * recovery loop provision a duplicate). A double-provision loser — * job bound to a DIFFERENT node — stays idle for the reaper; * - only jobs in `pre` or `active` (matching wantsPlacementNow and the * daemon's own provisionJob validation — an `active` job whose node * died gets re-provisioned and needs this bind exactly like a fresh * one; observed live 2026-08-14: a launch-failed job stayed `active`, * a `pre`-only guard skipped the replacement node, and the job hung). * Genuinely-running actives are covered by the binding guard above and * the daemon's ProvisionJobDuplicate check; * - never while a migration owns the job's lifecycle. * the binding is (re)confirmed synchronously before the await so the * hydration recovery loop can't re-place the job under the dispatch. * The daemon rejects duplicates (`ProvisionJobDuplicate`), so a re-fire * is an error surfaced via onError, not a double-start. */ private bindProvisionedJob; /** A startJob dispatch to a bound node failed. Unlike a launch rejection * (the worker refused after starting to launch), this is the daemon or * transport refusing the dispatch itself — historically it only bumped a * counter and fired a context-free onError, leaving the job bound to a * node that never ran it: the node sat idle until the elastic sweeper * reaped it (5-min dead zone), and the failure never counted toward the * restart ceiling, so a persistent dispatch failure looped forever * instead of terminating in a Failed bundle (observed live 2026-08-25). * * Now: release the binding, surface the error WITH job+node context, and * re-place through the recovery path — which runs the restart-rate * ceiling, so a job that can never dispatch trips `shouldBreakOnRestart` * → markJobFailed → BundleFailed rather than spinning. Excludes the node * we couldn't reach from the immediate re-place. */ private handleDispatchFailure; private handleNodeStopping; /** * The recovery a lost node needs, whichever event reported the loss: * `NodeStopping` normally, or a `NodeStopped` with no `NodeStopping` * before it (the daemon emits the two from separate sites). Drops the * node as placeable capacity, resolves migrations touching it, and * re-places every job still bound to it. */ private recoverLostNode; /** * The daemon changed a node's tags. Mostly one thing: a hot spare * was given a job and is a spare no longer — the daemon drops `spare` * the moment a job is bound to the node (task 90, the spare-pool churn: * the daemon's empty-node sweep used to reap every spare five minutes * after it came up, because a spare IS an empty elastic node, and the * reconciler replaced it, cold, for ever; now a node tagged `spare` is * exempt from the sweep, and ordinary once consumed). The metadata is * adopted whole, and a spare that stopped being one is a shortfall the * reconciler fills straight away — unless it was a LOCAL spare, which the * reconciler never counted, so the reconcile it runs finds nothing short. * The other thing it is: an operator marking or clearing a local server * as a spare (`setLocalSpare`), whose tags land the same way. */ private handleNodeTagsUpdated; private handleNodeStopped; private dropNodeServicePlacementsOnNode; private updateNodeMap; } export declare function getJobIdForNode(nodeMetadata: NodeMetadata | undefined): JobId | undefined; export declare function currentNodeIdForJob(jh: JobWithHistory | undefined): NodeId | undefined; /** * The job's current instance, from history: where it is (or was most * recently placed) and the instance KEY it wears there. Prefers the * most recent `running` over the most recent `provisioned` / `prepared` * — a job provisioned, run, stopped and re-provisioned but not yet run * again reports the newest provisioned node, but a running instance * always wins. `prepared` ranks with `provisioned`: containers exist, * none run. * * Wire history is NEWEST-first (JobStore prepends), so the scan starts * at index 0 and the first `running` met IS the most recent one. (The * earlier oldest-first walk returned the OLDEST running node — the same * answer for a job that has only ever lived on one node, and the wrong * one for a job that has moved: recovery, or a completed migration.) * * The role matters because after a migration the running instance is * `primary\@xxxxxxxx`, not `primary`, and that is the key the Manager * routes StopJob and the handshake by. */ export declare function currentInstanceForJob(jh: JobWithHistory | undefined): { nodeId: NodeId; role: Role; event: "running" | "provisioned" | "prepared"; } | undefined; //# sourceMappingURL=automanager.d.ts.map