import * as plugins from '../ts/plugins.js'; import { MigrationLeaseLostError } from '../ts/skr.errors.js'; import { SkrMigrationRecord, type TSkrMigrationReport } from './classes.migrationrecord.js'; /** What a running step is handed besides the database. */ export interface ISkrMigrationRunContext { /** * Aborted once the step's lease passed to another process or can no longer * be renewed before it runs out; its reason is a `MigrationLeaseLostError`. * A step calls `signal.throwIfAborted()` before every write or transaction. */ signal: AbortSignal; } /** * One versioned migration step. `run` must be safe to resume after a crash at * any point: every write is idempotent or commits together with the marker * that makes a rerun skip it. */ export interface ISkrMigrationStep { id: string; description: string; run(db: plugins.nosqldb.SmartdataDb, context: ISkrMigrationRunContext): Promise; } export type TSkrMigrationStepOutcome = 'applied' | 'already_applied'; /** How long a lease lasts, and how often its holder renews it. */ export interface ISkrMigrationLeaseTiming { leaseMs: number; renewMs: number; } /** A holder renews its lease while it works; a crashed holder's lease runs out. */ const DEFAULT_LEASE_TIMING: ISkrMigrationLeaseTiming = { leaseMs: 60_000, renewMs: 15_000 }; const POLL_MS = 500; const sleep = (ms: number): Promise => new Promise((resolve) => { setTimeout(resolve, ms); }); const isUniqueConflict = (error: unknown): boolean => error instanceof plugins.nosqldb.SmartdataPersistenceError && error.code === 'unique_conflict'; /** * Takes the step's lease, or learns that the step is applied. A lease another * process holds is waited out: it either finishes the step or stops renewing. */ const acquire = async ( step: ISkrMigrationStep, owner: string, timing: ISkrMigrationLeaseTiming, ): Promise<{ status: 'acquired'; leaseUntil: number } | { status: 'already_applied' }> => { for (;;) { const now = Date.now(); const existing = await SkrMigrationRecord.getInstance({ id: step.id }); if (!existing) { const record = new SkrMigrationRecord(); record.id = step.id; record.description = step.description; record.status = 'running'; record.owner = owner; record.leaseUntil = now + timing.leaseMs; record.startedAt = new Date(now); record.appliedAt = null; record.report = null; try { await SkrMigrationRecord.insert(record); return { status: 'acquired', leaseUntil: record.leaseUntil }; } catch (error) { if (!isUniqueConflict(error)) throw error; continue; } } if (existing.status === 'applied') { return { status: 'already_applied' }; } if (existing.leaseUntil < now) { const leaseUntil = now + timing.leaseMs; const taken = await SkrMigrationRecord.atomicUpdate( { id: step.id, status: 'running', owner: existing.owner, leaseUntil: existing.leaseUntil }, { $set: { owner, leaseUntil, startedAt: new Date(now) } }, ); if (taken.matchedCount === 1) { return { status: 'acquired', leaseUntil }; } continue; } await sleep(POLL_MS); } }; /** * Runs one step at most once per database, under a lease stored in its * record, so concurrent startups converge: one migrates, the others wait and * find it applied. The holder renews the lease while the step runs; once the * lease passes to another process, or cannot be renewed before it runs out, * the step's signal aborts and the step stops before its next write. The * report is persisted with the record by an owner-fenced write. */ export const runMigrationStep = async ( db: plugins.nosqldb.SmartdataDb, step: ISkrMigrationStep, timing: ISkrMigrationLeaseTiming = DEFAULT_LEASE_TIMING, ): Promise => { const owner = `skr-${process.pid}-${plugins.smartunique.shortId()}`; const lease = await acquire(step, owner, timing); if (lease.status === 'already_applied') { return 'already_applied'; } const controller = new AbortController(); const loseLease = (detail: string): void => { if (!controller.signal.aborted) { controller.abort(new MigrationLeaseLostError(step.id, detail)); } }; let confirmedUntil = lease.leaseUntil; let renewal: Promise = Promise.resolve(); const renewTimer = setInterval(() => { const leaseUntil = Date.now() + timing.leaseMs; renewal = SkrMigrationRecord.atomicUpdate( { id: step.id, status: 'running', owner }, { $set: { leaseUntil } }, ).then( (result) => { if (result.matchedCount === 1) { confirmedUntil = leaseUntil; } else { loseLease('passed to another process'); } }, () => { // a missed renewal is survivable while the confirmed lease outlasts // the next attempt; otherwise stop before the lease can run out if (Date.now() + timing.renewMs >= confirmedUntil) { loseLease('could not be renewed before it runs out'); } }, ); }, timing.renewMs); try { const report = await step.run(db, { signal: controller.signal }); clearInterval(renewTimer); await renewal; const recorded = await SkrMigrationRecord.atomicUpdate( { id: step.id, status: 'running', owner }, { $set: { status: 'applied', appliedAt: new Date(), report, leaseUntil: 0 } }, ); if (recorded.matchedCount !== 1) { throw new MigrationLeaseLostError(step.id, 'passed to another process before the step was recorded'); } return 'applied'; } catch (error) { clearInterval(renewTimer); await renewal; // hand the lease back at once, so the next start retries without waiting it out try { await SkrMigrationRecord.atomicUpdate( { id: step.id, status: 'running', owner }, { $set: { leaseUntil: 0 } }, ); } catch (releaseError) { throw new AggregateError( [error, releaseError], `Migration ${step.id} failed, and its lease could not be handed back (it runs out after ${timing.leaseMs} ms).`, ); } throw error; } };