// "Where is job J" — one record per job, in place of the five maps the // AutoManager used to keep in step by hand (`jobToNode`, `activeRole`, // `placementsByJob`, `lastPlacementAttemptAt`, and the migration index with // its target binding and parked target placement). The rules that were // restated at a dozen sites — "delete activeRole whenever jobToNode is // rebound", "the target binding folds into jobToNode at done" — are the // methods here, and `assertConsistent` says what must hold between them. import type { JobId, NodeId, Role } from "@norskvideo/norsk-manager-sdk"; import type { MigrationId, PlacementDecisionInfo } from "./automanager"; /** * The claim an in-flight migration holds for its job: which migration, and * — once placed — the node its TARGET is bound to and the target's own * placement decision, parked until `done` makes it the job's placement of * record. Deliberately separate from the job's own `nodeId`, which keeps * naming the source for the whole migration (that is where the job lives * until `done`): recovery, `nodesByJob` and `wantsPlacementNow` must never * mistake the target's existence for the job having moved, while the * derived reservation counts the job on BOTH nodes for as long as the * overlap lasts. * * @public */ export interface MigrationClaim { migrationId: MigrationId; targetNodeId?: NodeId; targetPlacement?: PlacementDecisionInfo; } /** * Everything the AutoManager knows about where one job is. * * @public */ export interface JobBinding { jobId: JobId; /** * The node the job's placement of record is on. Set at placement time * (an existing node via StartJob, or the provisional node of a fresh * provision), by hydration from job history, and by a migration's * `done`; released on rejection, dispatch failure, runtime failure, node * loss and job deletion. A node that has since terminated may still be * named here until the job is re-placed — resolve through the node map. */ nodeId?: NodeId; /** * The instance KEY of the job's current instance, where it differs from * the slot (`roleFromReplicaIndex`): set at a migration's `done` to the * target's key, rebuilt from history at hydration, and dropped with the * binding — a fresh instance is under the slot key again. Every RPC that * addresses the RUNNING instance goes through it: after one migration * that is `primary@xxxxxxxx`, not `primary`. Never set without `nodeId`. */ role?: Role; /** * The placement decision of record. Kept across an unbind — a node that * bounced does not change where the job was placed — and replaced by * the next successful placement; failures never bind one. */ placement?: PlacementDecisionInfo; /** * When the last initial placement was dispatched: the in-flight gate * that stops `handleJobChanged` re-firing placement while the daemon's * provisionJob is still mid-dispatch. Cleared whenever the attempt is * known to have concluded — a failure, an unbind — so the next capacity * event is free to retry. */ lastPlacementAttemptAt?: Date; /** The in-flight migration's claim, if one is in flight. */ migration?: MigrationClaim; } /** * The map of job bindings, with the operations the AutoManager needs and * nothing that can leave two fields disagreeing. * * @public */ export class JobBindings { private readonly byJob: Map = new Map(); // ---- reads ---------------------------------------------------------- get(jobId: JobId): JobBinding | undefined { return this.byJob.get(jobId); } nodeOf(jobId: JobId): NodeId | undefined { return this.byJob.get(jobId)?.nodeId; } roleOf(jobId: JobId): Role | undefined { return this.byJob.get(jobId)?.role; } placementOf(jobId: JobId): PlacementDecisionInfo | undefined { return this.byJob.get(jobId)?.placement; } lastAttemptOf(jobId: JobId): Date | undefined { return this.byJob.get(jobId)?.lastPlacementAttemptAt; } migrationOf(jobId: JobId): MigrationClaim | undefined { return this.byJob.get(jobId)?.migration; } /** Is this job's placement of record on `nodeId`? */ isBoundTo(jobId: JobId, nodeId: NodeId): boolean { return this.byJob.get(jobId)?.nodeId === nodeId; } /** Is this job's in-flight migration TARGET bound to `nodeId`? */ isTargetOn(jobId: JobId, nodeId: NodeId): boolean { return this.byJob.get(jobId)?.migration?.targetNodeId === nodeId; } /** Jobs whose placement of record is `nodeId`. A migration target bound * there is not yet on the node's books. */ jobsOn(nodeId: NodeId): JobId[] { const out: JobId[] = []; for (const b of this.byJob.values()) if (b.nodeId === nodeId) out.push(b.jobId); return out; } /** Jobs whose in-flight migration target is bound to `nodeId`. */ targetsOn(nodeId: NodeId): JobId[] { const out: JobId[] = []; for (const b of this.byJob.values()) if (b.migration?.targetNodeId === nodeId) out.push(b.jobId); return out; } /** Does any job's placement of record name `nodeId`? */ anyOn(nodeId: NodeId): boolean { for (const b of this.byJob.values()) if (b.nodeId === nodeId) return true; return false; } /** Every node some job's placement of record names. */ boundNodes(): Set { const out = new Set(); for (const b of this.byJob.values()) if (b.nodeId !== undefined) out.add(b.nodeId); return out; } /** Every (job, node) placement of record. */ bound(): Array<{ jobId: JobId; nodeId: NodeId }> { const out: Array<{ jobId: JobId; nodeId: NodeId }> = []; for (const b of this.byJob.values()) if (b.nodeId !== undefined) out.push({ jobId: b.jobId, nodeId: b.nodeId }); return out; } /** Every (job, node) in-flight migration target binding. */ targets(): Array<{ jobId: JobId; nodeId: NodeId; migrationId: MigrationId }> { const out: Array<{ jobId: JobId; nodeId: NodeId; migrationId: MigrationId }> = []; for (const b of this.byJob.values()) { if (b.migration?.targetNodeId !== undefined) { out.push({ jobId: b.jobId, nodeId: b.migration.targetNodeId, migrationId: b.migration.migrationId }); } } return out; } /** Every placement of record. */ placements(): PlacementDecisionInfo[] { const out: PlacementDecisionInfo[] = []; for (const b of this.byJob.values()) if (b.placement) out.push(b.placement); return out; } /** The id of every in-flight migration. */ inFlightMigrationIds(): MigrationId[] { const out: MigrationId[] = []; for (const b of this.byJob.values()) if (b.migration) out.push(b.migration.migrationId); return out; } // ---- writes --------------------------------------------------------- /** * A fresh placement of `jobId` onto `nodeId`: a new instance under the * slot key, so any key a past migration left the job under is forgotten * with it. `placement` (when the decision is known) becomes the placement * of record; the in-flight gate is left as the attempt set it. */ place(jobId: JobId, nodeId: NodeId, placement?: PlacementDecisionInfo): void { const b = this.ensure(jobId); b.nodeId = nodeId; delete b.role; if (placement) b.placement = placement; } /** * The provisioned node a job was waiting for has arrived: bind to it, * keeping whatever instance key hydration established — the instance * is the one the history describes, not a fresh one. */ adopt(jobId: JobId, nodeId: NodeId): void { this.ensure(jobId).nodeId = nodeId; } /** * Rebuilt from job history at hydration: the node the current instance * is on, the key it wears where that differs from the slot, and a * synthesised placement of record when one could be derived. */ hydrate(jobId: JobId, nodeId: NodeId, role: Role | undefined, placement: PlacementDecisionInfo | undefined): void { const b = this.ensure(jobId); b.nodeId = nodeId; if (role !== undefined) b.role = role; else delete b.role; if (placement) b.placement = placement; } /** * The job no longer runs where it was bound (or the attempt to put it * there conclusively failed): drop the node, the instance key and the * in-flight gate. The placement of record stays — it is what was * decided, and the next successful placement replaces it. A migration * claim is the saga's to release. */ unbind(jobId: JobId): void { const b = this.byJob.get(jobId); if (!b) return; delete b.nodeId; delete b.role; delete b.lastPlacementAttemptAt; this.prune(b); } /** `unbind`, but only when the placement of record is `nodeId` — the * guard every rejection / dispatch-failure path applies, in case * something re-placed the job meanwhile. Returns whether it did. */ releaseFrom(jobId: JobId, nodeId: NodeId): boolean { if (!this.isBoundTo(jobId, nodeId)) return false; this.unbind(jobId); return true; } /** The job is gone: forget everything about it. */ forget(jobId: JobId): void { this.byJob.delete(jobId); } /** An initial placement is being dispatched now. */ noteAttempt(jobId: JobId, at: Date): void { this.ensure(jobId).lastPlacementAttemptAt = at; } /** The attempt concluded (a failure, a superseding event): the next * capacity event may retry at once. */ clearAttempt(jobId: JobId): void { const b = this.byJob.get(jobId); if (!b) return; delete b.lastPlacementAttemptAt; this.prune(b); } /** Every in-flight gate, at once (close: nothing is in flight any more). */ clearAttempts(): void { for (const b of [...this.byJob.values()]) this.clearAttempt(b.jobId); } // ---- migrations ----------------------------------------------------- /** A migration of `jobId` is now in flight. One at a time per job: the * caller checks `migrationOf` first. */ claimMigration(jobId: JobId, migrationId: MigrationId): void { this.ensure(jobId).migration = { migrationId }; } /** The migration's target has been placed onto `nodeId`. */ bindTarget(jobId: JobId, migrationId: MigrationId, nodeId: NodeId): void { const m = this.claimFor(jobId, migrationId); if (m) m.targetNodeId = nodeId; } /** The target's placement decision, parked until `done`. */ parkTargetPlacement(jobId: JobId, migrationId: MigrationId, placement: PlacementDecisionInfo): void { const m = this.claimFor(jobId, migrationId); if (m) m.targetPlacement = placement; } /** The target lost its node (a provision abandoned, a rejection) while * the migration goes on: drop the target binding only. */ unbindTarget(jobId: JobId, migrationId: MigrationId): void { const m = this.claimFor(jobId, migrationId); if (m) delete m.targetNodeId; } /** * The migration is over without moving the job (aborted, or forgotten): * release its claim — target binding and parked placement included. * The job's own binding still names the source. A claim that is no * longer this migration's (a later one took over) is left alone. */ releaseMigration(jobId: JobId, migrationId: MigrationId): void { const b = this.byJob.get(jobId); if (!b || b.migration?.migrationId !== migrationId) return; delete b.migration; this.prune(b); } /** * `done`: the job now lives on the target. The binding rebinds to it, * the target's separate binding folds away, the instance key becomes * the target's, and the parked target decision becomes the placement of * record. Returns false when this migration held no claim. */ completeMigration(jobId: JobId, migrationId: MigrationId, targetRole: Role): boolean { const b = this.byJob.get(jobId); if (!b || b.migration?.migrationId !== migrationId) return false; const { targetNodeId, targetPlacement } = b.migration; if (targetNodeId !== undefined) b.nodeId = targetNodeId; b.role = targetRole; if (targetPlacement) b.placement = targetPlacement; delete b.migration; return true; } // ---- lifecycle ------------------------------------------------------ clear(): void { this.byJob.clear(); } /** * What must hold between the fields of every record, checked by tests * after every step of every scenario. Throws naming the first violation. */ assertConsistent(): void { for (const b of this.byJob.values()) { if (b.role !== undefined && b.nodeId === undefined) { throw new Error(`job ${b.jobId}: an instance key (${b.role}) without a node`); } if (b.nodeId === undefined && b.lastPlacementAttemptAt !== undefined && b.migration === undefined) { // An attempt still in flight is bound (the claim precedes the // dispatch); one that concluded has been cleared. throw new Error(`job ${b.jobId}: an in-flight placement attempt with nothing bound`); } if (b.migration !== undefined && b.nodeId === undefined) { throw new Error(`job ${b.jobId}: migration ${b.migration.migrationId} in flight for an unbound job`); } if (b.migration?.targetNodeId !== undefined && b.migration.targetNodeId === b.nodeId) { throw new Error(`job ${b.jobId}: migration target bound to the source node ${b.nodeId}`); } if (b.placement !== undefined && b.placement.jobId !== b.jobId) { throw new Error(`job ${b.jobId}: placement of record belongs to ${b.placement.jobId}`); } } } // ---- internals ------------------------------------------------------ private ensure(jobId: JobId): JobBinding { let b = this.byJob.get(jobId); if (!b) { b = { jobId }; this.byJob.set(jobId, b); } return b; } private claimFor(jobId: JobId, migrationId: MigrationId): MigrationClaim | undefined { const m = this.byJob.get(jobId)?.migration; return m?.migrationId === migrationId ? m : undefined; } /** A record with nothing left in it is not kept: `get` answering a * bare `{ jobId }` for a job nobody has placed would read as state. */ private prune(b: JobBinding): void { if ( b.nodeId === undefined && b.role === undefined && b.placement === undefined && b.lastPlacementAttemptAt === undefined && b.migration === undefined ) { this.byJob.delete(b.jobId); } } }