/** * System update orchestrator (CELILO_UPDATE Phase 4). * * Walks the dependency graph in topological order (providers first) * and drives each drifting module through: * * pending → backup → upgrade → deploy → health → done | failed | skipped * * Failure handling per D3: when a provider fails, downstream consumers * are checked against the *running* provider's `provides[X].version` * — if the consumer's new manifest can be satisfied by the running * version, the consumer proceeds; otherwise it's skipped with a * clear "blocked by old provider X@" reason. * * Every operation that can fail or take time is dependency-injected * (backup, upgrade, deploy, health). The orchestrator stays a pure * state machine the unit tests can drive without touching disk or * the network. */ import { runAudit } from '../audit'; import type { AuditDeps, SystemAuditReport } from '../audit'; import { type ModuleGraph, type ModuleId, topologicalOrder, transitiveConsumers, } from './dep-graph'; import type { ProgressEmitter } from './progress'; import type { SelfUpdateDeps } from './self-update'; import { performSelfUpdate } from './self-update'; import type { ModuleUpdateState, SelfUpdateResult, SystemUpdateResult, UpdateStep } from './types'; export interface ModuleSnapshot { id: string; /** Currently installed version (from DB). */ installedVersion: string; /** Latest version available in the registry (or null if not in registry). */ latestVersion: string | null; /** * Capability versions the *currently installed* manifest provides, * keyed by capability name. Used by the version-aware skip logic. */ installedProvides: Record; /** * Capability versions the *new* manifest (latest registry version) * requires. Used by the version-aware skip logic. */ pendingRequires: Record; } export interface OrchestratorOps { /** Take a pre-update backup for a module. Returns ok/error. */ backup: (moduleId: string, updateId: string) => Promise<{ ok: boolean; error?: string }>; /** Pull a newer version from the registry, write code/manifest, preserve state. */ upgrade: (moduleId: string) => Promise<{ ok: boolean; error?: string }>; /** Re-converge the deployed instance (terraform apply + ansible). */ deploy: (moduleId: string) => Promise<{ ok: boolean; error?: string }>; /** Run the module's health_check hook, report status. */ health: (moduleId: string) => Promise<{ status: 'healthy' | 'degraded' | 'unhealthy' | 'no-checks' | 'error'; detail?: string; }>; /** Read the celilo DB into a single-file snapshot — the framework-state half of D7. */ snapshotCeliloDb: (updateId: string) => Promise<{ ok: boolean; error?: string }>; } export interface RunUpdateDeps { audit: AuditDeps; graph: ModuleGraph; /** Map of moduleId → ModuleSnapshot for every module the orchestrator might touch. */ snapshots: Map; ops: OrchestratorOps; selfUpdate: SelfUpdateDeps; /** ID generator — defaults to crypto.randomUUID. Tests inject a fixed value. */ idGen?: () => string; /** Clock — defaults to `() => new Date()`. */ now?: () => Date; /** Progress callback for live UI / test capture. */ progress: ProgressEmitter; /** When true, skip backups and the celilo-db snapshot. Confirmed at the CLI layer. */ noBackup?: boolean; /** When true, allow destructive terraform plans through (D6). */ allowDestructive?: boolean; /** Restrict the run to one module + its dependencies. */ onlyModule?: string; } const VERSION_MAJOR_RE = /^v?(\d+)/; function majorOf(version: string): number { const m = version.match(VERSION_MAJOR_RE); return m ? Number.parseInt(m[1], 10) : 0; } /** * Decide whether a consumer's pending upgrade can proceed given the * provider's current (still-installed) version. * * Returns null if compatible (proceed); otherwise returns the reason * the consumer should be skipped. */ export function consumerSkipReason( consumer: ModuleSnapshot, failedProvider: ModuleSnapshot, graph: ModuleGraph, ): string | null { // Find which capabilities the consumer requires from the failed provider. const sharedCaps: string[] = []; for (const [cap, requiredVersion] of Object.entries(consumer.pendingRequires)) { const providerCurrentVersion = failedProvider.installedProvides[cap]; if (providerCurrentVersion === undefined) continue; // not a shared cap sharedCaps.push(cap); const requiredMajor = majorOf(requiredVersion); const providerMajor = majorOf(providerCurrentVersion); if (requiredMajor !== providerMajor) { return `blocked by ${failedProvider.id}@${failedProvider.installedVersion}: ${cap}@${providerCurrentVersion} doesn't satisfy ${cap}@${requiredVersion}`; } } // No shared capability between this consumer and the failed provider — // the dep-graph has them connected for some other capability that // didn't appear in `pendingRequires`. Be conservative: skip. if (sharedCaps.length === 0) { // We only got here because the dep-graph said `consumer → provider`, // so SOMETHING ties them. If pendingRequires doesn't include that // capability, treat as blocked (we can't reason about compatibility). void graph; return `blocked by ${failedProvider.id}: cannot verify compatibility`; } return null; // compatible; consumer can proceed against still-installed provider } /** * Drive a single module through the state machine. Stops at the first * failed step. */ async function runModule( moduleId: string, snap: ModuleSnapshot, deps: RunUpdateDeps, updateId: string, ): Promise { const startedAt = (deps.now ?? (() => new Date()))().toISOString(); const fromVersion = snap.installedVersion; const toVersion = snap.latestVersion ?? fromVersion; deps.progress.emit({ kind: 'module-start', moduleId, fromVersion, toVersion }); const finishWith = ( step: UpdateStep, extra: { error?: string; skipReason?: string } = {}, ): ModuleUpdateState => { const finishedAt = (deps.now ?? (() => new Date()))().toISOString(); const state: ModuleUpdateState = { moduleId, fromVersion, toVersion, step, startedAt, finishedAt, ...extra, }; if (step === 'failed') deps.progress.emit({ kind: 'module-failed', state }); else if (step === 'skipped') deps.progress.emit({ kind: 'module-skipped', state }); else deps.progress.emit({ kind: 'module-done', state }); return state; }; // 1. backup (per-module) if (!deps.noBackup) { deps.progress.emit({ kind: 'module-step', moduleId, step: 'backup' }); const r = await deps.ops.backup(moduleId, updateId); if (!r.ok) { return finishWith('failed', { error: `backup: ${r.error ?? 'unknown'}` }); } } // 2. upgrade if (snap.latestVersion && snap.latestVersion !== snap.installedVersion) { deps.progress.emit({ kind: 'module-step', moduleId, step: 'upgrade' }); const r = await deps.ops.upgrade(moduleId); if (!r.ok) { return finishWith('failed', { error: `upgrade: ${r.error ?? 'unknown'}` }); } } // 3. deploy deps.progress.emit({ kind: 'module-step', moduleId, step: 'deploy' }); const dr = await deps.ops.deploy(moduleId); if (!dr.ok) { return finishWith('failed', { error: `deploy: ${dr.error ?? 'unknown'}` }); } // 4. health deps.progress.emit({ kind: 'module-step', moduleId, step: 'health' }); const hr = await deps.ops.health(moduleId); if (hr.status === 'unhealthy' || hr.status === 'error') { return finishWith('failed', { error: `health: ${hr.status}${hr.detail ? ` (${hr.detail})` : ''}`, }); } return finishWith('done'); } /** * Top-level update flow. */ export async function runSystemUpdate(deps: RunUpdateDeps): Promise { const idGen = deps.idGen ?? (() => crypto.randomUUID()); const now = deps.now ?? (() => new Date()); const updateId = idGen(); const startedAt = now().toISOString(); const states: ModuleUpdateState[] = []; deps.progress.emit({ kind: 'plan', modules: deps.snapshots.size }); // Audit first; bail if BLOCKED. const audit: SystemAuditReport = await runAudit(deps.audit); if (audit.verdict === 'BLOCKED') { return { version: 1, updateId, startedAt, finishedAt: now().toISOString(), audit, selfUpdate: { performed: false, reason: 'no-network' }, backupsCreated: false, modules: [], ok: false, }; } // Self-update. deps.progress.emit({ kind: 'self-update-start' }); let selfUpdate: SelfUpdateResult; try { selfUpdate = await performSelfUpdate(deps.selfUpdate); } catch (err) { deps.progress.emit({ kind: 'self-update-skipped', reason: err instanceof Error ? err.message : String(err), }); return { version: 1, updateId, startedAt, finishedAt: now().toISOString(), audit, selfUpdate: { performed: false, reason: 'no-network' }, backupsCreated: false, modules: [], ok: false, }; } if (selfUpdate.performed) { deps.progress.emit({ kind: 'self-update-done', from: selfUpdate.from, to: selfUpdate.to, }); } else { deps.progress.emit({ kind: 'self-update-skipped', reason: selfUpdate.reason }); } // celilo-db snapshot (D7). let backupsCreated = false; if (!deps.noBackup) { const r = await deps.ops.snapshotCeliloDb(updateId); if (!r.ok) { // DB snapshot is the safety net for this whole run; if we can't // take it, refuse to mutate. return { version: 1, updateId, startedAt, finishedAt: now().toISOString(), audit, selfUpdate, backupsCreated: false, modules: [], ok: false, }; } backupsCreated = true; } // Walk modules in topological order. const order = topologicalOrder(deps.graph); const failed = new Map(); for (const moduleId of order) { if (deps.onlyModule && deps.onlyModule !== moduleId) { // Allow dependencies of `onlyModule` to be considered later if needed, // but for V1 the simple case is "just this one module". continue; } const snap = deps.snapshots.get(moduleId); if (!snap) continue; // no snapshot → not currently installed → skip // Check upstream failures first. const upstreamProviders = [...(deps.graph.edges.get(moduleId) ?? [])]; const blockingProvider = upstreamProviders .map((id) => failed.get(id)) .find((s): s is ModuleUpdateState => s !== undefined); if (blockingProvider) { const failedSnap = deps.snapshots.get(blockingProvider.moduleId); const reason = failedSnap ? consumerSkipReason(snap, failedSnap, deps.graph) : `blocked by ${blockingProvider.moduleId}: snapshot missing`; if (reason) { const state: ModuleUpdateState = { moduleId, fromVersion: snap.installedVersion, toVersion: snap.latestVersion ?? snap.installedVersion, step: 'skipped', skipReason: reason, startedAt: now().toISOString(), finishedAt: now().toISOString(), }; deps.progress.emit({ kind: 'module-skipped', state }); states.push(state); // Also propagate blockage to consumers of *this* skipped module. for (const id of transitiveConsumers(deps.graph, moduleId)) { if (!failed.has(id)) failed.set(id, state); } failed.set(moduleId, state); continue; } // version-compatible: proceed. } const state = await runModule(moduleId, snap, deps, updateId); states.push(state); if (state.step === 'failed') { failed.set(moduleId, state); } } const ok = states.every((s) => s.step === 'done' || s.step === 'pending'); deps.progress.emit({ kind: 'run-done', ok }); return { version: 1, updateId, startedAt, finishedAt: now().toISOString(), audit, selfUpdate, backupsCreated, modules: states, ok, }; }