/** * Module remove command * * Removes a module, destroying infrastructure if present. * Requires confirmation for infrastructure modules unless --force is passed. */ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import { eq, ne } from 'drizzle-orm'; import { FuelGauge } from '../../cli/fuel-gauge'; import { getDb } from '../../db/client'; import { capabilities as capabilitiesTable, moduleInfrastructure, modules } from '../../db/schema'; import { createGaugeLogger } from '../../hooks/logger'; import { runNamedHook } from '../../hooks/run-named-hook'; import { deallocateForModule } from '../../ipam/auto-allocator'; import { type ModuleManifest, ModuleManifestSchema } from '../../manifest/schema'; import { deleteMonitorForModule } from '../../services/alerting/monitors'; import { executeBuildWithProgress } from '../../services/build-stream'; import { askConfirm, withInterviewSession } from '../../services/bus-interview'; import { emitUninstallCompleted, emitUninstallFailed, emitUninstallStarted, } from '../../services/celilo-events'; import { PRE_DEPLOY_STATES, loadConsumerCleanupPlan, runConsumerCleanup, } from '../../services/consumer-cleanup'; import { getContainerService, getServiceCredentials } from '../../services/container-service'; import { completeOperation, failOperation, startOperation } from '../../services/module-operations'; import { type DependentCandidate, describeRemovalRefusal, findRemovalBlockers, } from '../../services/remove-guard'; import { getArg, hasFlag, validateRequiredArgs } from '../parser'; import { log } from '../prompts'; import type { CommandResult } from '../types'; /** * Handle module remove command * * Usage: celilo module remove [--force] * * @param args - Command arguments * @param flags - Command flags * @returns Command result */ export async function handleModuleRemove( args: string[], flags: Record = {}, ): Promise { // Validate arguments const error = validateRequiredArgs(args, 1); if (error) { return { success: false, error: `${error}\n\nUsage: celilo module remove [--force]`, }; } const moduleId = getArg(args, 0); if (!moduleId) { return { success: false, error: 'Module ID is required', }; } const force = hasFlag(flags, 'force'); const db = getDb(); // Check if module exists const module = db.select().from(modules).where(eq(modules.id, moduleId)).get(); if (!module) { return { success: false, error: `Module not found: ${moduleId}`, }; } // Check if any other installed module requires a capability this module provides const providedCapabilities = db .select() .from(capabilitiesTable) .where(eq(capabilitiesTable.moduleId, moduleId)) .all(); if (providedCapabilities.length > 0) { const providedNames = providedCapabilities.map((c) => c.capabilityName); const otherModules = db.select().from(modules).where(ne(modules.id, moduleId)).all(); const candidates: DependentCandidate[] = []; for (const m of otherModules) { const parsed = ModuleManifestSchema.safeParse(m.manifestData); if (!parsed.success) continue; candidates.push({ id: m.id, manifest: parsed.data, paused: m.state === 'PAUSED', deployed: !PRE_DEPLOY_STATES.has(m.state), }); } const blockers = findRemovalBlockers(providedNames, candidates); if (blockers.length > 0) { return { success: false, error: describeRemovalRefusal(moduleId, blockers), }; } } // Pre-flight passed; we're committed to attempting the uninstall. Start // operation tracking + emit lifecycle events from here on so backups and // restores see the uninstall as in-flight and downstream subscribers can // react. const startedAt = Date.now(); const opId = startOperation(moduleId, 'uninstall'); emitUninstallStarted({ module: moduleId, startedAt }); let result: CommandResult; try { result = await performModuleRemove(moduleId, module, force, db); } catch (err) { failOperation(opId, err); emitUninstallFailed({ module: moduleId, startedAt, durationMs: Date.now() - startedAt, error: err instanceof Error ? err.message : String(err), }); throw err; } const durationMs = Date.now() - startedAt; if (result.success) { completeOperation(opId); emitUninstallCompleted({ module: moduleId, startedAt, durationMs }); } else { failOperation(opId, result.error ?? 'unknown error'); emitUninstallFailed({ module: moduleId, startedAt, durationMs, error: result.error ?? 'unknown error', }); } return result; } /** * The actual uninstall work — on_uninstall hook, terraform destroy, DNS * deregister, IPAM deallocate, subscription cleanup, DB delete. Split out * so `handleModuleRemove` can wrap the call with operation tracking + * lifecycle event emission. Pre-conditions (module exists, no dependents) * are validated by the caller. */ async function performModuleRemove( moduleId: string, module: typeof modules.$inferSelect, force: boolean, db: ReturnType, ): Promise { // Run on_uninstall hook (if defined) BEFORE terraform destroy. Hooks // typically need the module's runtime to still be up so they can talk // to remote services — e.g. caddy's teardown SSHes the caddy host to // unexpose its ports + clear the Caddyfile, which can't happen after // terraform has destroyed the LXC. Failures here are best-effort: we // log the error and continue with removal so an operator can still // unstick a half-broken module. Use --force to skip the confirmation // prompt on hook failure. const manifestForHook = module.manifestData as { hooks?: { on_uninstall?: unknown } } | undefined; if (manifestForHook?.hooks?.on_uninstall) { // Match build-stream's TTY detection: in non-TTY contexts (tests, pipes, // CI) skipAnimation prevents the gauge's setInterval and raw-stdin // handlers from blocking process exit. const isInteractive = process.stdout.isTTY && process.stdin.isTTY; const gauge = new FuelGauge(`${moduleId}: on_uninstall`, { skipAnimation: !isInteractive, }); gauge.start(); const logger = createGaugeLogger(gauge, moduleId, 'on_uninstall'); const hookResult = await runNamedHook(moduleId, 'on_uninstall', db, logger, {}); if (hookResult.success) { gauge.stop(true); } else { gauge.stop(false); const reason = hookResult.error ?? 'unknown error'; if (!force) { // STOP, and leave the module ERRORED. Do not ask. // // `on_uninstall` is what withdraws a module's cross-module state — // caddy unexposes its ports and clears its Caddyfile, a firewall // provider withdraws its forwards. If it failed, that state is still // out there and we do not know how much of it the hook managed before // dying. Deleting the module now orphans whatever is left, with no // record of what to go clean up. // // This previously asked "continue removing anyway?" in a TTY and // silently continued when not one. Both were wrong. The prompt offers // the orphaning decision at the least informed possible moment — inside // a Y/N, before anyone has looked at the machine — and the non-TTY // default made orphaning the norm for every scripted removal. // // The module is marked ERROR rather than left reading INSTALLED, // because the previous behaviour recorded the failure only in a // `module_operations` row: `module list` and `system doctor` both // showed the module as perfectly healthy while its teardown had failed. // // The way forward is deliberate: go look, clean up by hand, then // re-run with --force, which means "I have remediated; drop the // record." db.update(modules) .set({ state: 'ERROR', errorMessage: `on_uninstall failed: ${reason}`, updatedAt: new Date(), }) .where(eq(modules.id, moduleId)) .run(); return { success: false, error: [ `Cannot remove '${moduleId}': its on_uninstall hook failed, so cross-module state it registered (port forwards, DNS records, web routes) may still exist.`, '', ` ${reason}`, '', `'${moduleId}' is now marked ERROR and has NOT been removed. Inspect what the hook left behind, clean it up, then re-run:`, ` celilo module remove ${moduleId} --force`, '', '--force means "I have remediated this by hand; delete the record anyway".', ].join('\n'), }; } log.warn(`on_uninstall failed: ${reason}`); log.info( '--force set: continuing removal. Any state the hook left behind is yours to clean up.', ); } } // Tell every provider whose capability this module consumed that it is // leaving, so each withdraws what it minted on its behalf // (openspec/changes/consumer-removal-cleanup). BEFORE terraform destroy, so // provider hosts are still reachable; AFTER on_uninstall, so a module that // tears its own state down first still wins. // // This replaces the by-name `cleanupWebRoutesForModule` call, which did the // same job for exactly one capability. A failed withdrawal never blocks the // removal — the failing PROVIDER is marked ERROR instead (D6). const cleanupLogger = { info: (m: string) => log.info(m), warn: (m: string) => log.warn(m), error: (m: string) => log.warn(m), success: (m: string) => log.info(m), }; const cleanup = await runConsumerCleanup( moduleId, loadConsumerCleanupPlan(moduleId, module.manifestData as ModuleManifest, db), db, cleanupLogger, ); if (cleanup.failures.length > 0) { log.warn( `${cleanup.failures.length} provider(s) could not withdraw state for '${moduleId}' and are now marked ERROR: ${cleanup.failures.map((f) => f.providerId).join(', ')}. Removal continues; run \`celilo system audit\` to see what each is holding.`, ); } // Check if module has infrastructure that needs to be destroyed const infra = db .select() .from(moduleInfrastructure) .where(eq(moduleInfrastructure.moduleId, moduleId)) .get(); const terraformDir = join(module.sourcePath, 'generated', 'terraform'); const hasTerraformState = existsSync(join(terraformDir, 'terraform.tfstate')); if (infra && hasTerraformState) { // Module has infrastructure — need to destroy it if (!force) { const confirmed = await withInterviewSession(() => askConfirm({ scope: `module-remove:${moduleId}`, key: 'destroy_infrastructure', message: `Module '${moduleId}' has deployed infrastructure. This will run terraform destroy to remove it. Continue?`, defaultValue: false, }), ); if (!confirmed) { return { success: false, error: 'Removal cancelled', }; } } // Build Terraform env vars (need service credentials for destroy) const terraformEnvVars: Record = { TF_IN_AUTOMATION: '1', }; if (infra.infrastructureType === 'container_service' && infra.serviceId) { const service = await getContainerService(infra.serviceId); if (service) { const credentials = await getServiceCredentials(infra.serviceId); if (service.providerName === 'digitalocean' && 'api_token' in credentials) { terraformEnvVars.TF_VAR_digitalocean_token = credentials.api_token; } else if (service.providerName === 'proxmox' && 'api_url' in credentials) { terraformEnvVars.TF_VAR_proxmox_api_url = credentials.api_url; terraformEnvVars.TF_VAR_proxmox_token_id = credentials.api_token_id; terraformEnvVars.TF_VAR_proxmox_token_secret = credentials.api_token_secret; } } } // Run terraform destroy log.info(`Destroying infrastructure for ${moduleId}...`); const initResult = await executeBuildWithProgress({ command: 'terraform', args: ['init', '-upgrade'], cwd: terraformDir, title: 'Initializing Terraform', env: terraformEnvVars, }); if (!initResult.success) { return { success: false, error: `Terraform init failed: ${initResult.error}`, }; } const destroyResult = await executeBuildWithProgress({ command: 'terraform', args: ['destroy', '-auto-approve', '-no-color'], cwd: terraformDir, title: 'Destroying infrastructure', env: terraformEnvVars, }); if (!destroyResult.success) { return { success: false, error: `Terraform destroy failed: ${destroyResult.error}\n\nInfrastructure may still exist. Check your provider console.`, }; } log.success('Infrastructure destroyed'); } // Capture the deployed systems BEFORE the module_systems rows are deleted // (cascade on module removal) so the system.destroyed events (D5/D6) carry // the host/IP each subscriber needs to deregister. One event per system. const removedSystems = await (async () => { try { const { getModuleSystems } = await import('../../services/deployed-systems'); return getModuleSystems(moduleId, db); } catch { return []; } })(); // Announce teardown on the bus (D5, openspec/specs/event-driven-hook-subscriptions/spec.md). A // dns_internal provider's system.destroyed.* subscription runs its // on_system_event hook (op: deregister) to remove each host's records. if (removedSystems.length > 0) { try { const { emitSystemDestroyed } = await import('../../services/celilo-events'); for (const sys of removedSystems) { emitSystemDestroyed({ module: moduleId, hostname: sys.hostname, targetIp: sys.ipv4_address, }); } } catch (error) { const msg = error instanceof Error ? error.message : String(error); log.warn(`Failed to emit system.destroyed for ${moduleId}: ${msg}`); } } // Deallocate IPAM resources (if any) await deallocateForModule(moduleId, db); // Tear down any event-bus subscriptions the module owned. Cascade-safe: // best-effort, never blocks removal — a stuck bus is no reason to // strand a module in a half-removed state. try { const { unregisterModuleSubscriptions } = await import('../../services/module-subscriptions'); const result = unregisterModuleSubscriptions(moduleId); if (result.unregistered > 0) { log.info(`Unregistered ${result.unregistered} event-bus subscription(s) for ${moduleId}`); } } catch (error) { const msg = error instanceof Error ? error.message : String(error); log.warn(`Failed to unregister event-bus subscriptions: ${msg}`); } // Drop the module's health monitor. No cascade can reach it — `monitors.target` // holds a module id or an audit check name depending on `kind`, so the column // carries no foreign key (celilo#1029). Left behind, the monitor fires // `Module not found` and then becomes permanently unschedulable, because its // cadence resolves from a module row that no longer exists — so nothing ever // runs it again to resolve the alert it just raised. // // What it was holding is named, not just counted. The alerts are deleted with // it (cascade), so a firing check goes silent and the coverage of it goes at // the same moment — an operator who is told only `removed monitor` never // learns which real failure just stopped being reported. try { const droppedAlerts = deleteMonitorForModule(db, moduleId); if (droppedAlerts) { log.info(`Removed health monitor for ${moduleId}`); for (const alert of droppedAlerts) { log.warn(` dropped live alert ${alert.key}: ${alert.message}`); } if (droppedAlerts.length > 0) { log.warn(` nothing checks ${moduleId} any more, so this will not be reported again.`); } } } catch (error) { const msg = error instanceof Error ? error.message : String(error); log.warn(`Failed to remove health monitor for ${moduleId}: ${msg}`); } // Delete module (cascade will remove configs, secrets, capabilities, infrastructure records) db.delete(modules).where(eq(modules.id, moduleId)).run(); return { success: true, message: `Successfully removed module: ${moduleId}`, }; }