import type * as ManagerPB from "@norskvideo/norsk-api/lib/manager_pb"; import type { NodeId, NorskManager } from "@norskvideo/norsk-manager-sdk"; import type { AutoSettings, HotSpareConfig, HotSpareSpecsDiff, NodeSummary, SpareSpec } from "./automanager"; import type { Clock } from "./clock"; import type { JobBindings } from "./jobBinding"; import { HeldImage, ImageRef, NodeView } from "./placement"; import type { BundleJobSpec, NodeInventory as TypedNodeInventory } from "./types"; /** How long a spare's failed prefetch waits before the reconciler asks * again. Five minutes: long enough that a registry outage or a stale * credential is not hammered, short enough that a fixed one is picked up * without an operator's nudge. */ export declare const PREFETCH_RETRY_BACKOFF_MS: number; /** * How long a pool's spare provisioning waits after a provision fails — the * create is refused, or the node stops before it ever runs. Every failure * used to reconcile straight back into another create: norsk-e2e * 2026-09-15, an AMI still `pending`, 29 RunInstances calls in 77 s. The * periodic tick retries once the backoff has passed. * @public */ export declare const SPARE_PROVISION_RETRY_BACKOFF_MS: number; /** What the hot-spare collaborator needs from the AutoManager, handed in * explicitly. `settings` is the live object: `updateHotSpareSpecs` writes * the new list into it, which is what every other reader sees. */ export interface HotSpareDeps { settings: AutoSettings; clock: Clock; norsk: Pick; bindings: JobBindings; /** False before hydration and after close: no reconcile runs then. */ active: () => boolean; nodes: () => Iterable; nodeOf: (nodeId: NodeId) => NodeSummary | undefined; inventoryOf: (nodeId: NodeId) => TypedNodeInventory | undefined; poolNameForNode: (nodeId: NodeId) => string; /** Record a provision as a `starting` node (and tell the embedder). */ nodeStarting: (nodeId: NodeId) => void; /** Drop a node that never came to exist (a create that failed). */ deleteNode: (nodeId: NodeId) => void; /** Apply a spec edit's tags to the node map now, ahead of the daemon's announcement. */ mergeNodeTags: (nodeId: NodeId, tags: Record) => void; /** The AutoManager's terminate path (fires onNodeTerminated once per node). */ issueTerminate: (nodeId: NodeId) => Promise; } export declare class HotSpares { private readonly deps; private readonly pendingSpareProvisions; private readonly warm; private readonly runtimeTemplateImages; private readonly sparePrefetched; private readonly spareProvisionFailedAt; private readonly sparePrefetchFailedAt; constructor(deps: HotSpareDeps); /** A pending spare provision landed (`nodeStarted`). */ spareLanded(nodeId: NodeId): void; /** A node stopped. A spare that stops while still pending never ran: its * create failed, and the replacement waits out the backoff rather than * repeating the failure. */ spareStopped(nodeId: NodeId): void; /** Hydration: the fleet view is being rebuilt from the initial state. */ clear(): void; /** * Phase C5 — is the named node currently tagged as a hot spare? * Spares are warm-running nodes provisioned (by the C6 reconciler) * with `NodeMetadata.tags["spare"] = "true"`. * * There is deliberately no per-spare dedication. A `forPolicy` field and a * matching `spareBand` node tag used to exist, but placement never read * them — they only partitioned the reconciler's own accounting — so a spare * "reserved" for one policy was reserved from nobody. Which work prefers or * avoids a spare is decided on the JOB, by its onRuntimeFailure. * @public */ isHotSpareNode(nodeId: NodeId): boolean; /** * A LOCAL SERVER (cluster-provider node) that is a hot spare — marked by * an operator through `setLocalSpare`, not provisioned by the reconciler * (review 2026-09-16, e2e-14). For placement it is a spare like any * other: prefer / avoid / reservation / warmth read the tag and the * running-jobs list, and consumption is the daemon dropping the tag. * For the RECONCILER it is nobody's: never counted toward an elastic * entry's target, never reaped as excess or as an orphan, never replaced * when consumed (there is nothing to provision), never re-specced by a * `hotSpares` edit. Every reconciler read goes through * `spareNodesMatching` / `reapOrphanedSpares`, which skip these. * @public */ isLocalSpare(nodeId: NodeId): boolean; private isLocalServer; /** * Register (or replace) the container images a product template needs. * Overlays `settings.templateImages`; the embedder that reads the * template manifest calls this when a template is added or rebuilt. * @public */ setTemplateImages(templateName: string, images: string[]): void; /** * The images a product template needs — runtime registration first, then * settings. Unknown ⇒ empty, which means "no warmth signal" rather than * "needs nothing": `warmth` scores 0 for every node, so placement is * unbiased rather than wrong. * @public */ templateImages(templateName: string): string[]; /** The template a placed bundle job launches, when it launches one. */ /** * The name warmth and reservation key a job's template by: the product * TEMPLATE's registered name (`cmaf-cdn`), which is what `warmFor` and * `reservedFor` name and what the operator registers image sets under. * Keying by product (`norsk-studio`) — as this did until 2026-09-15 — * made every template of a product one "template": a reservation could * not tell cmaf-cdn from studio-dev, and a warm list pulled the union of * every template's images (task 90, fix-list 45). A ref written before * the name existed carries none and keys by product, as before. */ templateNameFor(spec: BundleJobSpec | undefined): string | undefined; /** The image set placement should score warmth against, for one job. */ jobImagesFor(spec: BundleJobSpec | undefined): ImageRef[]; /** What a node advertises holding, through the G1 adapter. */ heldImagesFor(nodeId: NodeId): HeldImage[]; /** * A spare's live spec, read from its node tags: what it was provisioned * with, as edited since through `updateHotSpareNode`. The tags are the * only copy — the daemon persists an edit and re-applies it when it * rediscovers the node, so the spec survives a reconnect and a restart * on either side (task 90 fix-list 46: an in-memory override here was * lost on every norsk-mgr restart). A node that is not a spare has no * spec. * @public */ spareSpec(nodeId: NodeId): SpareSpec | undefined; /** The reservation half of a node's placement view. */ reservationViewFor(nodeId: NodeId): Pick; /** * Edit a running spare's spec. Adding to `warmFor` starts the pulls * (Steve's case: two spares warmed for `xyz`, a migration of `abc` * needed, so `abc` is added and the pull happens while the migration is * still proposing). Removing from it does NOT delete images — it only * stops us claiming the node is kept warm for that template; the images * stay until the node's own eviction reclaims them. * * Omitted fields keep their current value. Throws if the node is not a * hot spare. * @public */ updateHotSpareNode(nodeId: NodeId, edit: { warmFor?: string[]; reservedFor?: string[]; exclusive?: boolean; }): Promise; /** * Replace the live `hotSpares` list without a reconnect. The embedder * used to reload settings for this, which tore the AutoManager down and * — since `close()` aborts every in-flight migration — killed whatever * was migrating (the IBC-demo landmine). Now: the list is validated as * it would be at boot (the `hotSpares` slice of `validateAutoSettings`, * plus one pool per entry, since the pool is the key) and rejected whole * with the codes; otherwise it becomes the live list and the reconciler * converges on it on its next tick, which runs at once. A removed entry's * unconsumed spares are reaped as orphans; an added one is provisioned; a * lowered target terminates the excess coldest first; a changed * `warmFor` / `reservedFor` / `exclusive` is written to the live spares' * tags (only the fields that changed, so a per-node edit of the others * survives), and the pulls follow. Nothing else is touched — a migration * in flight stays in flight. * @public */ updateHotSpareSpecs(specs: HotSpareConfig[]): Promise; /** * Mark a LOCAL SERVER as a hot spare, or clear it (review 2026-09-16, * e2e-14: "no reason why we wouldn't support a local server as a hot * spare"). The daemon owns the tag: it is asked through `setNodeTags` * with the same tag set an elastic spare is provisioned with (`spare`, * `warmFor`, `reservedFor`, `exclusive`), persists it, announces it, and * drops `spare` the moment a job binds — so from then on the node is a * spare exactly as an elastic one is, to placement and to norsk-mgr. * * Refused here, before the wire, for a node that is not a local server * (an elastic spare is the reconciler's to make) and for a node that * holds a job (a spare is empty; the daemon refuses the same, and the * local check also covers a placement this AutoManager has made that no * report has confirmed yet). `warmFor` / `reservedFor` / `exclusive` * omitted on a mark keep whatever the node's tags say; on a clear they * are left alone. The view is updated at once rather than waiting for * the daemon's announcement, and a mark starts the pulls for `warmFor`. * @public */ setLocalSpare(nodeId: NodeId, edit: { spare: boolean; warmFor?: string[]; reservedFor?: string[]; exclusive?: boolean; }): Promise; /** * Override how long this node lingers idle before the daemon expires it * (fix-list item 8: the node a handover vacated was destroyed before you * could migrate back onto it). `0` restores the global `emptyNodeLinger`. * @public */ setNodeIdleExpiry(nodeId: NodeId, ms: number): Promise; /** * A prefetch batch settled. Warmth itself comes from the node's * re-advertised `heldImages` (which follows this event), so nothing is * driven off it — it is logged, and handed to the operator, because a * ref that would not pull is exactly what somebody needs to be told: * the spare will stay cold for that template until it is fixed. */ onImagesPrefetched(raw: ManagerPB.ImagesPrefetchedEvent): void; /** * Send the node a prefetch for the images of `warmFor`'s templates, if * that set has changed since the last one we sent it. Idempotent, so * every reconcile can call it. */ /** * Pull the images of some product templates onto ANY node, spare or not * (task 90, fix-list 39). * * A spare's `warmFor` edit does this and also records the declaration; * an ordinary node — the one a handover just vacated, which is the fastest * target the next handover could have — has no spec to edit, but the pull * is the same pull, and warmth is OBSERVED (`heldImages`) so placement * scores it the moment the images land. Nothing is declared or reserved. * Resolves with the image refs asked for; empty when no template resolves * to any image (no warmth signal exists for it yet), which is not an * error. Rejects for a node this AutoManager does not know. * @public */ prefetchTemplates(nodeId: NodeId, templateNames: string[]): Promise; private ensurePrefetched; /** * Drive every configured `hotSpares` entry toward its `targetCount`. * Counts live + pending spares matching the pool; provisions * short, terminates excess (oldest live first). Cluster pools are * non-elastic — under-target just fires `onSpareUnderflow` and * leaves provisioning to the operator. Idempotent and cheap; safe * to call on every lifecycle event plus the periodic tick. */ reconcile(): void; /** * How warm a spare is for the work its pool exists to serve: the overlap * between what it advertises holding and the union of the images of the * pool entry's `warmFor` templates. * * When the entry declares no `warmFor`, there is no intended image set to * measure against, so the fallback is how much the node holds at all, * normalised into (0,1) so it can never outrank a genuine warmth score. * That keeps "coldest first" meaningful for an undeclared pool: an empty * node still goes before a laden one. */ private spareWarmth; /** * A spare whose pool no longer has a reserve entry (the entry was deleted, * or every entry was) has a target of zero and nobody else to reap it: * the daemon's empty-node sweep leaves a `spare`-tagged node alone by * design, and the per-entry loop below never looks at a pool it has no * entry for. Observed on norsk-e2e 2026-09-14: two spares outlived their * reserve indefinitely after DELETE /api/hot-spares. Terminate them. */ private reapOrphanedSpares; /** * The ELASTIC spares an entry governs: tagged, running, in its pool. A * local spare in the same pool is not one of them — it is not counted * toward the target (so it never displaces a provision), not excess (so * it is never terminated), and not re-specced by an entry edit. */ private spareNodesMatching; private pendingSparesMatching; private provisionHotSpare; } //# sourceMappingURL=hotSpares.d.ts.map