/** * System update command — orchestrates the audit-determined upgrade flow. * * Usage: * celilo system update [--module ] [--no-backup] [--allow-destructive] * [--dry-run] [--json] * * Per CELILO_UPDATE Phase 4: runs `system audit` first, refuses on * BLOCKED, then walks each drifting module through * backup → upgrade → deploy → health, isolating failures to the * dependency subtree. * * This is a thin adapter — per Rule 10.5: parse args, compose deps, * call `runSystemUpdate`, format output, return. The real logic * lives in `services/update/orchestrator.ts`. */ import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { eq } from 'drizzle-orm'; import { getDb } from '../../db/client'; import { backups, moduleConfigs as moduleConfigsTbl, modules } from '../../db/schema'; import type { ModuleManifest } from '../../manifest/schema'; import { RegistryClient } from '../../registry/client'; import { runAudit } from '../../services/audit'; import { loadAbandonedOperations } from '../../services/audit/abandoned-operations'; import { collectBrowserPinDeps } from '../../services/audit/browser-pin'; import { fetchLatestCliVersion } from '../../services/audit/cli-version'; import { unusedPublicDnsProbe } from '../../services/audit/public-dns'; import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema'; import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create'; import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule'; import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner'; import { collectJailExemptions } from '../../services/jail-exemptions'; import { configOverride, parseStoredConfigValue } from '../../services/module-config'; import { deployModule } from '../../services/module-deploy'; import { buildModuleGraph } from '../../services/update/dep-graph'; import { type ModuleSnapshot, type OrchestratorOps, runSystemUpdate, } from '../../services/update/orchestrator'; import type { SystemUpdateResult } from '../../services/update/types'; import { getFlag, hasFlag } from '../parser'; import type { CommandResult } from '../types'; import { fetchAndUpdate } from './module-update'; /** * Packages we manage via the global bun install. The self-update step * runs `bun update -g` against all of these in one shot, so operators * never have to remember the three-package incantation. * * Ordering matters cosmetically only — bun resolves them all together. */ const MANAGED_PACKAGES = ['@celilo/cli', '@celilo/event-bus', '@celilo/e2e'] as const; /** * Result of inspecting backup-storage configuration before letting * `runSystemUpdate` take a celilo-DB snapshot. Three failure shapes * the operator might hit, each with a distinct friendly message: * * 1. No storage rows at all → run `storage add local` * 2. Storage rows exist, none verified → run `storage verify ` * (the case the operator hits when an unverified storage was * left behind by a failed verify) * 3. Verified storage exists, none is default → run * `storage set-default ` * 4. Default exists but isn't verified → run `storage verify ` * * Pure data in / data out so it's testable without setting up an * audit, registry, or DB. */ export interface BackupStorageLike { storageId: string; verified: boolean; } export function checkBackupStoragePreflight(input: { defaultStorage: BackupStorageLike | null; allStorages: BackupStorageLike[]; }): { kind: 'ok' } | { kind: 'error'; message: string } { const { defaultStorage, allStorages } = input; if (defaultStorage) { if (defaultStorage.verified) return { kind: 'ok' }; return { kind: 'error', message: `Default backup storage '${defaultStorage.storageId}' is not verified. Run: celilo storage verify ${defaultStorage.storageId} Then re-run system update.`, }; } if (allStorages.length === 0) { return { kind: 'error', message: `No backup storage configured. celilo system update snapshots the celilo DB before applying module updates as a safety net. Configure backup storage first: celilo storage add local Or skip the safety net entirely (the CLI self-update still runs): celilo system update --no-backup`, }; } const verified = allStorages.filter((s) => s.verified); if (verified.length === 0) { const ids = allStorages.map((s) => s.storageId).join(', '); return { kind: 'error', message: `Backup storage exists but none is verified yet. Storages: ${ids} Verify one before re-running system update: celilo storage verify Common causes of failed verification: - The path requires elevated permissions (try a path under your home directory; the default '${join(homedir(), '.local/share/celilo/backups')}' works without sudo). - The disk is full or the mount point isn't writeable. Or skip the safety net entirely (the CLI self-update still runs): celilo system update --no-backup`, }; } const ids = verified.map((s) => s.storageId).join(', '); return { kind: 'error', message: `Backup storage is verified but no default is set. Verified: ${ids} Set a default before re-running system update: celilo storage set-default Or skip the safety net entirely (the CLI self-update still runs): celilo system update --no-backup`, }; } /** * Should `system update` take a celilo-DB snapshot for this run? * True iff at least one module that's currently DEPLOYED has a * registry-newer version waiting. Pure data in / data out. * * Excludes IMPORTED-but-not-yet-deployed modules — refreshing their * on-disk source has no impact on live state, so the safety net * isn't needed (and an operator who hasn't configured storage yet * shouldn't be blocked by a snapshot they don't need). */ export function shouldTakeBackup(input: { snapshots: Map; wasDeployed: Set; }): boolean { for (const [id, s] of input.snapshots) { if (!input.wasDeployed.has(id)) continue; if (!s.latestVersion) continue; if (s.latestVersion === s.installedVersion) continue; return true; } return false; } function readInstalledCliVersion(): string { const here = dirname(fileURLToPath(import.meta.url)); const candidates = [ join(here, '..', '..', '..', 'package.json'), join(process.cwd(), 'package.json'), ]; for (const path of candidates) { if (existsSync(path)) { try { const pkg = JSON.parse(readFileSync(path, 'utf-8')) as { version?: string }; if (pkg.version) return pkg.version; } catch { // try the next candidate } } } return '0.0.0'; } function findMigrationsFolderSafe(): string | null { try { const here = dirname(fileURLToPath(import.meta.url)); const candidates = [ join(here, '..', '..', '..', 'drizzle'), join(process.cwd(), 'drizzle'), join(process.cwd(), 'apps', 'celilo', 'drizzle'), ]; for (const c of candidates) { if (existsSync(join(c, 'meta', '_journal.json'))) return c; } return null; } catch { return null; } } /** * Build a snapshot map from installed modules + registry lookups. * * `installedProvides` and `pendingRequires` come from the manifest * — `installedProvides` from the currently-deployed manifest in the * DB; `pendingRequires` from the manifest of the latest registry * version (so the version-aware skip can compare what the new code * needs against what the still-running provider has). * * Phase 4 V1: we don't yet fetch the new manifest from the registry * — that needs a registry endpoint that exposes the embedded * manifest.yml. Until that lands, `pendingRequires` is populated * from the *currently installed* manifest (so consumer compatibility * checks behave as "are the running versions still compatible?"). * That degrades the skip logic to the pre-D3 behavior in mixed * cases but doesn't crash anything. */ async function buildSnapshots( db: ReturnType, installed: Array, registry: RegistryClient, ): Promise> { const snapshots = new Map(); for (const m of installed) { const manifest = m.manifestData as ModuleManifest; const provides: Record = {}; for (const p of manifest.provides?.capabilities ?? []) { provides[p.name] = p.version; } const requires: Record = {}; for (const r of manifest.requires?.capabilities ?? []) { requires[r.name] = r.version; } for (const r of manifest.optional?.capabilities ?? []) { requires[r.name] = r.version; } let latest: string | null = null; try { const entries = await registry.getIndex(m.id); const top = registry.latestVersion(entries); latest = top ? top.vers : null; } catch { // network error → leave null } snapshots.set(m.id, { id: m.id, installedVersion: m.version, latestVersion: latest, installedProvides: provides, pendingRequires: requires, }); } // Suppress the unused-db warning while we don't yet need the parameter. // (Future revs read pending manifests from the DB cache.) void db; return snapshots; } /** * Live ops — wraps the existing per-module services as the * orchestrator's hook callbacks. * * - `backup` calls `createModuleBackup`. Modules without an * `on_backup` hook return ok (nothing to back up; not an error). * - `upgrade` fetches the latest version from the registry and runs * the in-place upgrade (`fetchAndUpdate` from module-upgrade.ts). * Same code path that `module update` uses for its sweep mode. * - `deploy` calls `deployModule`. * - `health` calls `runModuleHealthCheck`. * - `snapshotCeliloDb` calls `createSystemStateBackup`. */ function buildOps( registry: RegistryClient, wasDeployed: Set, quiet: boolean, ): OrchestratorOps { return { backup: async (moduleId, _updateId) => { // IMPORTED modules don't have running state to back up; skip // silently. The orchestrator already gates on `!noBackup`, so // this is the second layer (per-module rather than system-wide). if (!wasDeployed.has(moduleId)) return { ok: true }; const db = getDb(); const mod = db.select().from(modules).where(eq(modules.id, moduleId)).get(); if (!mod) return { ok: false, error: `module ${moduleId} not found` }; const manifest = mod.manifestData as ModuleManifest; if (!manifest.hooks?.on_backup) { return { ok: true }; } const result = await createModuleBackup(moduleId); return result.success ? { ok: true } : { ok: false, error: result.error }; }, // Fetch the latest version from the registry and run the in-place // upgrade (file-copy + DB update + capability re-register, preserving // configs/secrets/infra). Reuses the same path `module update` runs // for its sweep mode, so behavior is consistent regardless of which // entry point the operator uses. upgrade: async (moduleId) => { const db = getDb(); try { const entries = await registry.getIndex(moduleId); if (entries.length === 0) { return { ok: false, error: `${moduleId}: not in registry` }; } const latest = registry.latestVersion(entries); if (!latest) { return { ok: false, error: `${moduleId}: no non-yanked version` }; } const result = await fetchAndUpdate(registry, moduleId, latest.vers, db, {}); if (result.status === 'success') return { ok: true }; if (result.status === 'failed') return { ok: false, error: result.error }; // 'skipped' isn't expected here (the module IS installed), but // surface it as a non-fatal so the orchestrator can decide. return { ok: false, error: `unexpected skip: ${result.reason}` }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err), }; } }, deploy: async (id) => { // IMPORTED modules weren't deployed before this run; we just // refreshed their source files via `upgrade`. Don't try to // deploy them here — that would surprise the operator who // hasn't asked for a deploy. They'll get a `module deploy ` // todo finding from the audit on the next run. if (!wasDeployed.has(id)) return { ok: true }; const db = getDb(); // The deploy interview runs through the bus; if config is // missing the deploy hangs waiting for a responder. system-update // runs against pre-staged modules — config gaps surface via // `celilo events list-pending` and are an operator concern. const result = await deployModule(id, db, {}); return result.success ? { ok: true } : { ok: false, error: result.error }; }, health: async (id) => { // Same gating as deploy: there's no live deployment to health- // check for an IMPORTED module that we just refreshed. Return // pass with a "not deployed" detail so the orchestrator's // module-step record doesn't say "health: skipped" (which would // imply a check ran and skipped its assertions). if (!wasDeployed.has(id)) { // The orchestrator's HealthStatus type uses 'healthy' / // 'degraded' / 'unhealthy' / 'error' / 'no-checks'. There's // no "skipped" — closest meaningful value for "we didn't // run the check on purpose" is 'no-checks' (semantically: // "the module didn't have any health checks to run"). return { status: 'no-checks', detail: 'not deployed; health check skipped' }; } // We can call the existing health-runner directly today — that's // already a clean service. quiet keeps the check's gauge and plain // log lines off stdout when the caller asked for JSON (celilo#1362). const db = getDb(); const r = await runModuleHealthCheck(id, db, { quiet }); return { status: r.status, detail: r.error }; }, snapshotCeliloDb: async (_updateId) => { // resolveStorage throws when no default backup storage is configured // (or it isn't verified). Catch here so the orchestrator gets a // clean { ok: false, error } shape — the alternative is the throw // escapes runSystemUpdate and surfaces as an unhandled stack trace. // The handleSystemUpdate pre-flight already catches the common // "no storage configured" case with a friendly message; this is // a belt-and-suspenders for any other throw path resolveStorage // takes (e.g. storage exists but isn't verified). try { const result = await createSystemStateBackup(); return result.success ? { ok: true } : { ok: false, error: result.error }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err), }; } }, }; } function formatResult(result: SystemUpdateResult): string { const lines: string[] = []; lines.push(''); // The updateId is for audit / journal correlation (see the // `backups` table's updateId FK); operators don't read it during // normal use, only when debugging a specific run via --json or // `celilo backup list`. Surfacing it in human output added noise // without paying for itself. lines.push(`System update ${result.ok ? 'completed' : 'FAILED'}`); lines.push(` audit verdict: ${result.audit.verdict}`); lines.push( ` self-update: ${result.selfUpdate.performed ? `${result.selfUpdate.from} → ${result.selfUpdate.to}` : `(${result.selfUpdate.reason})`}`, ); lines.push(` backups: ${result.backupsCreated ? 'created' : 'skipped'}`); // Surface audit findings inline. Three severity tiers: // blocked → ✗ gates the run (orchestrator short-circuits) // drift → ▸ informational; the system moved away from a // desired state, but the run still proceeded // todo → ➤ next-step reminders (e.g. "you imported X but // haven't deployed it yet"); never escalates the // verdict. // Each tier renders separately so operators can scan for what // needs attention vs. what's just a friendly nudge. if (result.audit.findings.length > 0) { lines.push(''); const blocked = result.audit.findings.filter((f) => f.severity === 'blocked'); const drift = result.audit.findings.filter((f) => f.severity === 'drift'); const todo = result.audit.findings.filter((f) => f.severity === 'todo'); if (blocked.length > 0) { lines.push(` BLOCKED (${blocked.length}):`); for (const f of blocked) { lines.push(` ✗ [${f.category}] ${f.subject}: ${f.message}`); if (f.remediation) lines.push(` → ${f.remediation}`); } } if (drift.length > 0) { lines.push(` Drift findings (${drift.length}, informational):`); for (const f of drift) { lines.push(` ▸ [${f.category}] ${f.subject}: ${f.message}`); if (f.remediation) lines.push(` → ${f.remediation}`); } } if (todo.length > 0) { lines.push(` Todos (${todo.length}, next-step reminders):`); for (const f of todo) { lines.push(` ➤ [${f.category}] ${f.subject}: ${f.message}`); if (f.remediation) lines.push(` → ${f.remediation}`); } } } if (result.modules.length > 0) { lines.push(''); for (const m of result.modules) { const tag = m.step === 'done' ? '✓' : m.step === 'failed' ? '✗' : m.step === 'skipped' ? '↳' : '?'; const change = m.fromVersion === m.toVersion ? m.fromVersion : `${m.fromVersion} → ${m.toVersion}`; lines.push(` ${tag} ${m.moduleId} (${change}) — ${m.step}`); if (m.error) lines.push(` ${m.error}`); if (m.skipReason) lines.push(` ${m.skipReason}`); } } return lines.join('\n'); } export async function handleSystemUpdate( _args: string[], flags: Record, ): Promise { const json = hasFlag(flags, 'json'); const dryRun = hasFlag(flags, 'dry-run'); const noBackup = hasFlag(flags, 'no-backup'); const allowDestructive = hasFlag(flags, 'allow-destructive'); const onlyModule = getFlag(flags, 'module', '') || undefined; const db = getDb(); const installed = db.select().from(modules).all(); // Include IMPORTED modules in the upgrade scope so a fresh // `module import ` followed (later, possibly weeks later) by // `system update` actually picks up registry-side updates for // not-yet-deployed modules. The deploy and health steps are gated // on prior state in buildOps so we don't deploy something the // operator hasn't asked us to deploy. // INSTALLED + VERIFIED + IMPORTED is the upgrade-eligible set; // ERROR / DEPLOYING / GENERATING / UNINSTALLING are skipped (the // operator needs to handle those manually first). const upgradeEligibleStates = new Set(['INSTALLED', 'VERIFIED', 'IMPORTED']); const upgradableModules = installed.filter((m) => upgradeEligibleStates.has(m.state)); // Track which modules were already deployed BEFORE this run. // Anything else gets the upgrade only — no deploy / health / backup // attempts (those would be no-ops at best, surprising at worst). const wasDeployed = new Set( installed.filter((m) => ['INSTALLED', 'VERIFIED'].includes(m.state)).map((m) => m.id), ); const registry = new RegistryClient(); // Build the dep graph from installed manifests. const graph = buildModuleGraph(upgradableModules.map((m) => m.manifestData as ModuleManifest)); const snapshots = await buildSnapshots(db, upgradableModules, registry); // Build audit deps. (Mostly mirrors system-audit.ts; could be refactored // into a shared helper in Phase 5.) const migrationsFolder = findMigrationsFolderSafe(); const healthResults = await runAllHealthChecks(db, { quiet: json }); // Module integrity, shallow. Local and milliseconds; the host plane is one // SSH per system and belongs to `module verify --deep`. const { auditModule } = await import('../../module/packaging/audit'); const integrityResults = await Promise.all(upgradableModules.map((m) => auditModule(m.id, db))); const latestBackupByModule = new Map(); try { const successfulBackups = db .select() .from(backups) .where(eq(backups.status, 'completed')) .all(); for (const b of successfulBackups) { if (!b.moduleId || !b.completedAt) continue; const prev = latestBackupByModule.get(b.moduleId); const ts = b.completedAt.getTime(); if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts); } } catch { // backups table missing on a fresh DB — fine. } const allConfigs = db.select().from(moduleConfigsTbl).all(); const configsByModule = new Map>(); for (const c of allConfigs) { const m = configsByModule.get(c.moduleId) ?? {}; m[c.key] = parseStoredConfigValue(c); configsByModule.set(c.moduleId, m); } const auditDeps = { cliVersion: { installedVersion: readInstalledCliVersion(), fetcher: fetchLatestCliVersion, }, schema: { journal: migrationsFolder ? makeJournalReader(migrationsFolder) : () => null, applied: readAppliedMigrations, db, }, capabilityAbi: { modules: upgradableModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, browserPin: collectBrowserPinDeps(upgradableModules), terraformPlan: { modules: upgradableModules.map((m) => ({ id: m.id, terraformDir: existsSync(join(m.sourcePath, 'generated', 'terraform')) ? join(m.sourcePath, 'generated', 'terraform') : null, })), run: async () => ({ exitCode: 0, stdout: '', stderr: '' }), }, moduleVersions: { installed: upgradableModules.map((m) => ({ id: m.id, version: m.version })), fetcher: async (id: string) => { const entries = await registry.getIndex(id); const top = registry.latestVersion(entries); return top ? { latest: top.vers } : { latest: null }; }, }, moduleConfigs: { modules: upgradableModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, configs: configsByModule.get(m.id) ?? {}, })), }, moduleIntegrity: { results: integrityResults }, detectWithoutConverge: { modules: upgradableModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, jailExemptions: { exemptions: collectJailExemptions(db) }, health: { results: healthResults }, backups: { modules: upgradableModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY), lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null, })), }, abandonedOperations: { records: loadAbandonedOperations(db) }, undeployedModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, errorMessage: m.errorMessage, })), }, unconfiguredModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, configCount: Object.keys(configsByModule.get(m.id) ?? {}).length, })), }, // The update flow doesn't run these checks itself — pass empty // results so the typed AuditDeps shape is satisfied. The full // audit (with credential + secret decryption checks) runs in // `system audit`; pre-update we just consume what we have. servicesCredentials: { results: [] }, secretsDecryptable: { results: [] }, servicesReachable: { results: [] }, machinesReachable: { results: [] }, // Public reachability needs a network round trip per name; the update // flow's partial audit does no probing. `system audit` and the scheduled // monitor own this check. publicDns: { records: [], probe: unusedPublicDnsProbe }, // Disk probing is one SSH per system; the update flow's partial audit // does no probing. `system audit` and the monitor sweep own this check. diskSpace: { results: [] }, transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 }, trustedSources: { firewalls: [], unreachableFirewalls: [] }, }; if (dryRun) { const audit = await runAudit(auditDeps); const plan = { version: 1 as const, updateId: crypto.randomUUID(), audit, willSelfUpdate: false, // computed at run time; the dry-run preview is best-effort modules: upgradableModules .filter((m) => { const snap = snapshots.get(m.id); return snap?.latestVersion && snap.latestVersion !== snap.installedVersion; }) .map((m) => ({ moduleId: m.id, fromVersion: m.version, toVersion: snapshots.get(m.id)?.latestVersion ?? m.version, dependsOn: [...(graph.edges.get(m.id) ?? [])], })), willBackup: !noBackup, destructiveTerraformBlocked: !allowDestructive, }; return json ? { success: true, message: JSON.stringify(plan, null, 2), rawOutput: true } : { success: true, message: `dry-run: ${plan.modules.length} module(s) would be updated, backups=${plan.willBackup ? 'on' : 'off'}, allow-destructive=${allowDestructive}\n${plan.modules .map((m) => ` ▸ ${m.moduleId}: ${m.fromVersion} → ${m.toVersion}`) .join('\n')}`, }; } // Decide whether the celilo-db snapshot is even needed for this run. // // The snapshot is a safety net — its purpose is to let `system update` // roll back if a deploy / health step bricks a running module. So // we ONLY need it when the run will actually upgrade-and-redeploy // a currently-deployed module. Two cases that don't qualify: // // 1. Nothing has new code waiting at all (everyone's at latest). // 2. The only modules with new code are IMPORTED-but-not-deployed. // Their upgrade is a pure source-files-on-disk refresh; no // live state to roll back, no risk to mitigate. Forcing a // backup here means an operator on a fresh celilo-mgmt with // nothing deployed yet has to configure backup storage just // to refresh the on-disk modules they imported — which is // exactly the friction that prompted this code path. const effectiveNoBackup = noBackup || !shouldTakeBackup({ snapshots, wasDeployed }); // Pre-flight the storage check so a missing/unusable default doesn't // reach the orchestrator's snapshot hook (where the throw would // surface as a hostile stack trace). Skip when we're already not // going to backup. if (!effectiveNoBackup) { const { getDefaultBackupStorage, listBackupStorages } = await import( '../../services/backup-storage' ); const preflight = checkBackupStoragePreflight({ defaultStorage: getDefaultBackupStorage(), allStorages: listBackupStorages(), }); if (preflight.kind === 'error') { return { success: false, error: preflight.message }; } } const ops = buildOps(registry, wasDeployed, json); const result = await runSystemUpdate({ audit: auditDeps, graph, snapshots, ops, selfUpdate: { installedVersion: readInstalledCliVersion(), fetcher: fetchLatestCliVersion, // Refresh ALL managed @celilo/* packages in one bun-update call. // The orchestrator only tracks @celilo/cli's from/to versions, // but we sweep event-bus and e2e along with it so operators // don't have to type the three-package incantation themselves. // Failures from `bun update` propagate as stderr; the orchestrator // surfaces them in the run summary. updater: async () => { const r = spawnSync('bun', ['update', '-g', ...MANAGED_PACKAGES], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8', }); if (r.status === 0) return { ok: true, stderr: '' }; return { ok: false, stderr: (r.stderr ?? '').trim() || `bun update -g exited ${r.status}`, }; }, }, progress: { emit() {} }, noBackup: effectiveNoBackup, allowDestructive, onlyModule, }); // The audit baked into `result.audit` ran BEFORE the orchestrator // upgraded any modules. Its findings are now stale for whatever // we just upgraded — `module_versions` drift, `module_configs` // referencing the OLD manifest's required vars, `capability_abi` // checking pre-upgrade providers. Re-query the modules table and // re-run the audit so the displayed findings reflect post-upgrade // reality. Only do this on a successful run (a partial-failure // run keeps its original audit so the operator sees what the // orchestrator was reacting to). const successfulModuleSteps = result.modules.filter((m) => m.step === 'done'); if (result.ok && successfulModuleSteps.length > 0) { const refreshedAudit = await runAudit(await rebuildAuditDepsForRerun(auditDeps, db)); result.audit = refreshedAudit; } if (json) { if (result.ok) { return { success: true, message: JSON.stringify(result, null, 2), rawOutput: true }; } return { success: false, error: JSON.stringify(result, null, 2) }; } if (result.ok) { return { success: true, message: formatResult(result) }; } return { success: false, error: `${formatResult(result)}\n\none or more modules failed; see output above`, }; } type AuditDeps = Parameters[0]; /** * Build a fresh AuditDeps object whose module-state-dependent * sub-deps are re-queried from the DB. The non-module deps * (cliVersion fetcher, migrations, terraform plan runner, registry * fetcher, etc.) are reused from the original deps because they * don't change during a single system-update run. * * Used by the post-orchestrator re-audit so displayed findings * reflect post-upgrade reality (e.g., a module_versions drift * finding for a module we just upgraded is no longer reported). */ export async function rebuildAuditDepsForRerun( original: AuditDeps, db: ReturnType, ): Promise { const installed = db.select().from(modules).all(); const upgradeEligibleStates = new Set(['INSTALLED', 'VERIFIED', 'IMPORTED']); const upgradable = installed.filter((m) => upgradeEligibleStates.has(m.state)); const allConfigs = db.select().from(moduleConfigsTbl).all(); const configsByModule = new Map>(); for (const c of allConfigs) { const m = configsByModule.get(c.moduleId) ?? {}; m[c.key] = parseStoredConfigValue(c); configsByModule.set(c.moduleId, m); } const { auditModule } = await import('../../module/packaging/audit'); // Backup recency is unchanged across an orchestrator run (only // celilo-DB snapshots happen, not per-module backup writes), so // we look up each module's prior lastSuccessfulBackupAt by id // rather than re-querying the backups table. const priorBackupByModule = new Map(); for (const b of original.backups.modules) { priorBackupByModule.set(b.id, b.lastSuccessfulBackupAt); } return { ...original, capabilityAbi: { modules: upgradable.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, moduleVersions: { ...original.moduleVersions, installed: upgradable.map((m) => ({ id: m.id, version: m.version })), }, moduleConfigs: { modules: upgradable.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, configs: configsByModule.get(m.id) ?? {}, })), }, // Re-MEASURED, not carried over. An upgrade rewrites the installed tree and // the baseline, so the pre-upgrade result describes files that are no // longer on disk. Reusing it would be reporting a stored claim about a // state that has since changed, which is the exact mistake this whole // change exists to remove. moduleIntegrity: { results: await Promise.all(upgradable.map((m) => auditModule(m.id, db))), }, detectWithoutConverge: { modules: upgradable.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, // Unchanged across an orchestrator run — an upgrade does not reclaim // abandoned operations, so re-reading them would be the same rows. abandonedOperations: original.abandonedOperations, backups: { ...original.backups, modules: upgradable.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY), lastSuccessfulBackupAt: priorBackupByModule.get(m.id) ?? null, })), }, undeployedModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, errorMessage: m.errorMessage, })), }, unconfiguredModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, configCount: Object.keys(configsByModule.get(m.id) ?? {}).length, })), }, }; }