/** * `celilo module upgrade [name]` — the build-bus CD verb (ISS-0138) + the * registry-poll it drives (ISS-0139). openspec/changes/build-bus-poll-cd/proposal.md. * * module upgrade one module: registry-latest → update → [backup, * posture-gated] → deploy → verify. * module upgrade NO name = the CD poll: every INSTALLED module that * opted into `auto_upgrade` and has a newer registry * version, upgraded the same way. This is what the * timer-driven CD job runs. * * `module update` refreshes the stored definition; `module deploy` applies it; * `module upgrade` is the safe compound. Fast/safe posture is derived * (deploy-posture.ts): a low-risk revision/patch skips the backup, a minor/major * backs up first. */ import { eq } from 'drizzle-orm'; import { type DbClient, getDb } from '../../db/client'; import { modules } from '../../db/schema'; import type { ModuleManifest } from '../../manifest/schema'; import { RegistryClient } from '../../registry/client'; import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create'; import { getDefaultBackupStorage } from '../../services/backup-storage'; import { capabilityBlockersForManifest } from '../../services/capability-compat'; import { type DeployPosture, type UpgradePolicy, resolveDeployPosture, } from '../../services/deploy-posture'; import { runModuleHealthCheck } from '../../services/health-runner'; import { getModuleConfigValue } from '../../services/module-config'; import { deployModule } from '../../services/module-deploy'; import { getArg, getFlag } from '../parser'; import { log } from '../prompts'; import type { CommandResult } from '../types'; import { fetchTargetManifest } from './module-update'; import { classifyVersionChange, fetchAndUpdate } from './module-update'; const VALID_POLICIES: readonly UpgradePolicy[] = ['by-semver', 'always-safe', 'always-fast']; /** * Pick the effective upgrade policy. Pure (Rule 10): operator config decides; * an unknown/absent value falls back to `by-semver`. * * Config-only by construction — the manifest schema is `.strict()` and declares * no `upgrade_policy`, so a manifest that set one could never pass validation. * (The old manifest-default branch read a key no valid manifest can carry — * Rule 3.9 dead code, deleted rather than left as a corpse.) */ export function pickUpgradePolicy(fromConfig: string | undefined): UpgradePolicy { return VALID_POLICIES.includes(fromConfig as UpgradePolicy) ? (fromConfig as UpgradePolicy) : 'by-semver'; } /** * Resolve whether a module opts into auto-upgrade by the poll. Pure (Rule 10): * operator config only (same `.strict()` reason as pickUpgradePolicy), default * OFF — a production module isn't auto-upgraded unless the operator chose it: * `celilo module config set auto_upgrade true`. */ export function pickAutoUpgrade(fromConfig: string | boolean | undefined): boolean { if (typeof fromConfig === 'boolean') return fromConfig; return fromConfig === 'true'; } /** * Pure (Rule 10): does this upgrade need a pre-deploy backup? Yes IFF the * posture is "safe" (a minor/major bump) AND the TARGET version's manifest * declares an on_backup hook. * * The `targetManifest` MUST be the version being upgraded TO (read back after * the def is refreshed), never the installed one — gating on the installed * manifest skips the backup on the very upgrade that introduces the hook * (ISS-0168). */ export function needsPreUpgradeBackup( posture: DeployPosture, targetManifest: ModuleManifest, ): boolean { return posture === 'safe' && Boolean(targetManifest.hooks?.on_backup); } /** * Pure (Rule 10): does this upgrade snapshot the celilo DB first? Safe posture * means a minor/major bump, and a minor bump can migrate the DB schema — the * snapshot is what makes a later package downgrade a real rollback instead of * a gamble on an older binary reading a newer-migrated database. */ export function needsSystemStateSnapshot(posture: DeployPosture): boolean { return posture === 'safe'; } /** * Take the system_state snapshot a safe-posture upgrade requires. * * A host with no default backup storage keeps today's behavior (warn and * proceed): refusing every safe upgrade there would break CD for a config * gap, and the module-level backup below still runs when the target manifest * declares on_backup. A configured-but-failing snapshot refuses the upgrade, * same rule `system update` applies to its own snapshot (D7): the DB snapshot * is the safety net, and an upgrade that cannot take it should not mutate. */ async function snapshotSystemStateBeforeUpgrade( moduleId: string, ): Promise<{ ok: true } | { ok: false; error: string }> { try { if (!getDefaultBackupStorage()) { log.warn( `${moduleId}: no default backup storage configured — skipping pre-upgrade system_state snapshot`, ); return { ok: true }; } const snapshot = await createSystemStateBackup(); if (!snapshot.success) { return { ok: false, error: snapshot.error ?? 'system_state backup failed' }; } log.success(`System state snapshot taken before upgrading ${moduleId}`); return { ok: true }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err) }; } } export interface PollCandidate { moduleId: string; installed: string; /** Latest registry version, or null when the module isn't on the registry. */ latest: string | null; autoUpgrade: boolean; /** True when the module row carries state ERROR (a failed upgrade, celilo#1363). */ inError: boolean; } export interface PollTarget { moduleId: string; from: string; to: string; } /** * Pure: which candidates the poll should upgrade — opted into auto_upgrade AND a * newer registry version exists. (Rule 10; the I/O — registry queries — happens * in the caller.) */ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] { const targets: PollTarget[] = []; for (const c of candidates) { if (!c.autoUpgrade || !c.latest) continue; // A module in ERROR failed an upgrade already. Retrying it on every poll // redeploys the same broken version indefinitely (celilo#1363), so skip it // until an attended health check clears the state (healthy → VERIFIED). if (c.inError) continue; const change = classifyVersionChange(c.installed, c.latest); if (change === 'up-to-date' || change === 'ahead') continue; targets.push({ moduleId: c.moduleId, from: c.installed, to: c.latest }); } return targets; } /** Resolve a module's auto_upgrade opt-in from operator config. */ function resolveAutoUpgrade(moduleId: string): boolean { const cfg = getModuleConfigValue(moduleId, 'auto_upgrade'); return pickAutoUpgrade( typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined, ); } /** * Upgrade ONE module to a known target version: posture → update → backup → * deploy → verify. The shared core for both the single-module command and the * poll. `mod` is the current DB row; `targetVersion` is the registry latest. * Exported for tests: the failure paths here are where the store diverges from * reality (celilo#1363). */ export async function upgradeOneModule( mod: typeof modules.$inferSelect, targetVersion: string, client: RegistryClient, db: DbClient, flags: Record, ): Promise { const moduleId = mod.id; // GATE (celilo#1361): read the TARGET manifest BEFORE anything mutates, and // refuse an upgrade whose capability requirements no deployed provider // serves. Deploying first and failing in the module's publish hook is how // celilo-website burned six release revisions (+1 through +6): each failure // recorded the new version, re-triggered the poll, and changed nothing. // A deferral changes no state and retries once the provider lands. const targetManifestCheck = await fetchTargetManifest(client, moduleId, targetVersion); if (targetManifestCheck.ok) { const blockers = capabilityBlockersForManifest(db, targetManifestCheck.manifest); if (blockers.length > 0) { const why = blockers.map((b) => b.message).join('; '); return { success: false, deferred: true, error: `${moduleId} upgrade to ${targetVersion} deferred: the deployed providers cannot serve its capability requirements: ${why}. Upgrade the provider module first; the registry poll retries automatically.`, details: blockers, }; } } // A failed fetch here is NOT deferred — `fetchAndUpdate` below reports the // same failure through its own channel with the real error text. // Update FIRST (refresh stored def). Every downstream decision — the // backup gate and the posture policy — must consult the TARGET version's // manifest, not the installed one. Gating on the pre-update manifest skipped // the on_backup hook on the very upgrade that INTRODUCES it (e.g. a stateful // module that first ships backup support doesn't protect its own next // upgrade) — ISS-0168. const updated = await fetchAndUpdate(client, moduleId, targetVersion, db, flags); if (updated.status !== 'success') { const why = updated.status === 'failed' ? updated.error : updated.reason; return { success: false, error: `Update failed for ${moduleId}: ${why}` }; } // Read the target def back: fetchAndUpdate persisted it but doesn't return // the manifest. Fall back to the pre-update manifest only if the row somehow // vanished (it won't on the success path). const updatedRow = db.select().from(modules).where(eq(modules.id, moduleId)).get(); const targetManifest = (updatedRow?.manifestData as ModuleManifest | undefined) ?? (mod.manifestData as ModuleManifest); // Posture. Version delta is installed→target; the policy comes from operator // config (`celilo module config set upgrade_policy …`), else by-semver. const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy'); const modulePolicy = pickUpgradePolicy( typeof configPolicy?.value === 'string' ? configPolicy.value : undefined, ); // Per-release deploy_posture override lives in the .netapp release metadata; // reading it requires fetching the package first. Deferred — the classifier // supports it (deploy-posture.ts) and no app stamps it yet. const { posture, reason } = resolveDeployPosture({ installed: mod.version, next: targetVersion, releasePosture: null, modulePolicy, }); log.info(`Upgrading ${moduleId} ${mod.version} → ${targetVersion} (${posture} — ${reason})`); // Safe → snapshot the celilo DB first (the package rollback pair), then // back up the module itself, gated on the TARGET manifest's on_backup hook. if (needsSystemStateSnapshot(posture)) { const snapshot = await snapshotSystemStateBeforeUpgrade(moduleId); if (!snapshot.ok) { return { success: false, error: `Pre-upgrade system_state snapshot failed for ${moduleId}: ${snapshot.error}`, }; } } if (needsPreUpgradeBackup(posture, targetManifest)) { const backup = await createModuleBackup(moduleId); if (!backup.success) { return { success: false, error: `Pre-upgrade backup failed for ${moduleId}: ${backup.error}`, }; } log.success(`Backed up ${moduleId} before deploy`); } else if (posture === 'safe') { log.warn(`${moduleId} has no on_backup hook — proceeding without a pre-upgrade backup`); } // Deploy (idempotent). const deployed = await deployModule(moduleId, db, {}); if (!deployed.success) { // fetchAndUpdate already swapped the files and wrote the new version to // the store, so the recorded version no longer matches anything verified. // Record the failure in the state rather than leaving a stale VERIFIED // beside the new version (celilo#1363, option A). An attended health check // that passes moves ERROR → VERIFIED and clears the record. db.update(modules) .set({ state: 'ERROR', errorMessage: `Upgrade to ${targetVersion} failed to deploy: ${deployed.error}`, updatedAt: new Date(), }) .where(eq(modules.id, moduleId)) .run(); return { success: false, error: `Deploy failed for ${moduleId} (now at ${targetVersion}): ${deployed.error}`, }; } // Verify. const health = await runModuleHealthCheck(moduleId, db, {}); if (health.status === 'unhealthy' || health.status === 'error') { return { success: false, error: `Upgraded ${moduleId} to ${targetVersion} but post-deploy verify failed (health: ${health.status}${health.error ? ` — ${health.error}` : ''}).`, }; } const verifyNote = health.status === 'degraded' ? ' (health: degraded)' : ''; return { success: true, message: `Upgraded ${moduleId} ${mod.version} → ${targetVersion} (${posture}); verified${verifyNote}.`, }; } /** Latest registry version for a module, or null when absent. */ async function latestRegistryVersion( client: RegistryClient, moduleId: string, ): Promise { const entries = await client.getIndex(moduleId); return entries.length > 0 ? (client.latestVersion(entries)?.vers ?? null) : null; } /** * The CD poll (ISS-0139): upgrade every INSTALLED module that opted into * auto_upgrade and has a newer registry version. This is what the timer-driven * CD job runs (install-mode gating — ISS-0142 — applies at the scheduling layer). */ async function runRegistryPoll( db: DbClient, flags: Record, ): Promise { const client = new RegistryClient(getFlag(flags, 'registry', '') || undefined); const installed = db.select().from(modules).all(); const candidates: PollCandidate[] = []; const rowById = new Map(); for (const mod of installed) { rowById.set(mod.id, mod); candidates.push({ moduleId: mod.id, installed: mod.version, latest: await latestRegistryVersion(client, mod.id), autoUpgrade: resolveAutoUpgrade(mod.id), inError: mod.state === 'ERROR', }); } const targets = selectPollTargets(candidates); if (targets.length === 0) { return { success: true, message: 'Registry poll: all auto_upgrade modules are up to date.' }; } log.info(`Registry poll: ${targets.length} module(s) to upgrade.`); const upgraded: string[] = []; const deferred: string[] = []; const failed: string[] = []; // Serial, in installed order. (Strict provider-before-consumer ordering via // topologicalOrder is a refinement; the poll is idempotent + re-runs.) for (const t of targets) { const mod = rowById.get(t.moduleId); if (!mod) continue; const result = await upgradeOneModule(mod, t.to, client, db, flags); if (result.success) { upgraded.push(`${t.moduleId}→${t.to}`); } else if (result.deferred) { // Expected steady state while a provider upgrade is still pending: the // consumer retries next tick. Not a failure — celilo#1361. deferred.push(`${t.moduleId}: ${result.error}`); } else { failed.push(`${t.moduleId}: ${result.error}`); } } if (failed.length > 0) { return { success: false, error: `Registry poll: upgraded ${upgraded.length}, deferred ${deferred.length}, FAILED ${failed.length}:\n` + ` ${[...failed, ...(deferred.length > 0 ? [`(deferred) ${deferred.join('\n (deferred) ')}`] : [])].join('\n ')}`, }; } if (deferred.length > 0) { return { success: true, message: `Registry poll: upgraded ${upgraded.length}, deferred ${deferred.length} (capability requirements not yet served — retries next poll):\n ${deferred.join('\n ')}`, }; } return { success: true, message: `Registry poll: upgraded ${upgraded.length} — ${upgraded.join(', ')}.`, }; } /** * Pure (Rule 10.1): is this the CD poll rather than a single-module upgrade? * * `--poll` is what a bus subscription MUST use. The dispatcher spawns a * subprocess handler as ` ` (openspec/specs/event-bus/spec.md), * so a bare `celilo module upgrade` handler arrives as `celilo module upgrade * 5517` — the event id lands in the optional module-name slot and the poll dies * with "Module not found: 5517" every tick. An explicit flag makes the poll * invocation immune to the appended id instead of relying on argv position. */ export function isPollInvocation(args: string[], flags: Record): boolean { return Boolean(flags.poll) || !getArg(args, 0); } export async function handleModuleUpgrade( args: string[], flags: Record = {}, ): Promise { const db = getDb(); const moduleId = getArg(args, 0); // `--poll` or no name → the CD poll over all auto_upgrade modules. if (isPollInvocation(args, flags) || !moduleId) { return runRegistryPoll(db, flags); } const mod = db.select().from(modules).where(eq(modules.id, moduleId)).get(); if (!mod) { return { success: false, error: `Module not found: ${moduleId}` }; } const client = new RegistryClient(getFlag(flags, 'registry', '') || undefined); const latest = await latestRegistryVersion(client, moduleId); if (!latest) { return { success: false, error: `${moduleId} is not on the registry — nothing to upgrade to.` }; } const change = classifyVersionChange(mod.version, latest); if (change === 'up-to-date' || change === 'ahead') { return { success: true, message: `${moduleId} is already up to date (${mod.version}).` }; } return upgradeOneModule(mod, latest, client, db, flags); }