/** * System audit command — reports drift across the system. * * Usage: * celilo system audit # human-readable table * celilo system audit --json # machine-readable for scripts * * Read-only. No mutations. Wires the per-category audit checks * (`apps/celilo/src/services/audit/*`) to the live DB / registry / * health-runner / terraform binary, then formats the resulting * `SystemAuditReport` for stdout. * * The actual audit logic lives in services/audit/. This command is a * thin adapter — per Rule 10.5, it's a "thin adapter": parse args, * compose deps, call `runAudit`, format output, return. */ import { execFile } from 'node:child_process'; import { existsSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { isNotNull } from 'drizzle-orm'; import { testDigitalOceanConnection } from '../../api-clients/digitalocean'; import { testProxmoxConnection } from '../../api-clients/proxmox'; import { getDb } from '../../db/client'; import { capabilitySecrets, moduleConfigs as moduleConfigsTbl, modules, secrets, systemSecrets, } from '../../db/schema'; import type { ModuleManifest } from '../../manifest/schema'; import { RegistryClient } from '../../registry/client'; import { decryptSecret } from '../../secrets/encryption'; import { getOrCreateMasterKey } from '../../secrets/master-key'; import { readAllTransportStatuses } from '../../services/alerting/read-records'; import { runAudit } from '../../services/audit'; import type { DriftFinding, SystemAuditReport } from '../../services/audit'; import { loadAbandonedOperations } from '../../services/audit/abandoned-operations'; import { loadBackupAuditInfo } from '../../services/audit/backup-source'; import { collectBrowserPinDeps } from '../../services/audit/browser-pin'; import { type LatestCliVersionFetcher, fetchLatestCliVersion, } from '../../services/audit/cli-version'; import type { ModuleVersionFetcher } from '../../services/audit/module-versions'; import { loadPublicDnsEvidence, loadPublicDnsRecords, } from '../../services/audit/public-dns-source'; import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema'; import type { SecretCheckResult } from '../../services/audit/secrets-decryptable'; import type { ServiceCredentialsResult } from '../../services/audit/services-credentials'; import type { ServiceReachableResult } from '../../services/audit/services-reachable'; import type { TerraformPlanRunner } from '../../services/audit/terraform-plan'; import { getServiceCredentials, listContainerServices } from '../../services/container-service'; import { probeDiskUsage } from '../../services/disk-probe'; import { collectFirewallReach } from '../../services/firewall-reach'; import { runAllHealthChecks } from '../../services/health-runner'; import { collectJailExemptions } from '../../services/jail-exemptions'; import { probeMachines } from '../../services/machine-probe'; import { parseStoredConfigValue } from '../../services/module-config'; import { createPublicDnsProbe, loadPublicDnsProbeSettings } from '../../services/public-dns-probe'; import { buildTerraformEnvForModule } from '../../services/terraform-env'; import { hasFlag } from '../parser'; import type { CommandResult } from '../types'; const execFileAsync = promisify(execFile); /** Read the running CLI version from the bundled package.json. */ function readInstalledCliVersion(): string { const here = dirname(fileURLToPath(import.meta.url)); const candidates = [ join(here, '..', '..', '..', 'package.json'), join(process.cwd(), 'package.json'), ]; for (const path of candidates) { if (existsSync(path)) { try { const pkg = JSON.parse(readFileSync(path, 'utf-8')) as { version?: string }; if (pkg.version) return pkg.version; } catch { // try the next candidate } } } return '0.0.0'; // unknown — caller treats as "ahead of latest" } function makeRegistryFetcher(client: RegistryClient): ModuleVersionFetcher { return async (moduleId) => { const entries = await client.getIndex(moduleId); if (entries.length === 0) return { latest: null }; const latest = client.latestVersion(entries); if (!latest) return { latest: null }; return { latest: latest.vers, intermediateCount: entries.length }; }; } const realTerraformPlan: TerraformPlanRunner = async (terraformDir, envVars) => { try { const result = await execFileAsync( 'terraform', ['plan', '-detailed-exitcode', '-no-color', '-input=false'], { cwd: terraformDir, timeout: 120_000, // Inherit current env, then layer module-specific TF_VAR_* // credentials so terraform sees the same world `module deploy` // would. Without this, modules with provider creds (Proxmox, // DigitalOcean) fail immediately with "no value for required // variable" — a false positive, not real drift. env: { ...process.env, ...envVars }, }, ); return { exitCode: 0, stdout: result.stdout, stderr: result.stderr }; } catch (err) { const e = err as { code?: number; stdout?: string; stderr?: string; message?: string }; return { // detailed-exitcode: 0 = no diff, 1 = error, 2 = diff exitCode: e.code ?? 1, stdout: e.stdout ?? '', stderr: e.stderr ?? e.message ?? 'terraform plan failed', }; } }; function findMigrationsFolderSafe(): string | null { // Mirror db/client.ts:findMigrationsFolder, but return null instead // of throwing so the audit gracefully reports "schema check skipped" // rather than crashing. try { const here = dirname(fileURLToPath(import.meta.url)); const candidates = [ join(here, '..', '..', '..', 'drizzle'), join(process.cwd(), 'drizzle'), join(process.cwd(), 'apps', 'celilo', 'drizzle'), ]; for (const c of candidates) { if (existsSync(join(c, 'meta', '_journal.json'))) return c; } return null; } catch { return null; } } /** * Build the dependency object for `runAudit` from the live system. * * `onProgress` (when supplied) suppresses stdout-bound progress * indicators (FuelGauge) inside the audit subroutines and routes * progress messages to the caller instead — used by the TUI so * audit progress appears inside the alt-screen render. * * `quiet` (when true) silences them outright: the caller is going to * emit a JSON document on stdout, and a gauge's frames are not JSON * (celilo#1362). */ async function buildAuditDeps( options: { onProgress?: (msg: string) => void; quiet?: boolean } = {}, ) { const db = getDb(); const installed = db.select().from(modules).all(); const deployedModules = installed.filter((m) => ['INSTALLED', 'VERIFIED'].includes(m.state)); const registryClient = new RegistryClient(); // Build per-module config map for module-configs check. const allConfigs = db .select() .from(moduleConfigsTbl) .where(isNotNull(moduleConfigsTbl.moduleId)) .all(); const configsByModule = new Map>(); for (const c of allConfigs) { const map = configsByModule.get(c.moduleId) ?? {}; map[c.key] = parseStoredConfigValue(c); configsByModule.set(c.moduleId, map); } const installedConfigs = deployedModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, configs: configsByModule.get(m.id) ?? {}, })); const installedBackupInfo = loadBackupAuditInfo(db); // Compose per-module TF_VAR_* env vars in parallel — each call hits // the secret store / DB so they're not free, but they're all // independent. Empty record for machine-bound modules. const terraformEnvByModule = new Map>(); await Promise.all( deployedModules.map(async (m) => { try { terraformEnvByModule.set(m.id, await buildTerraformEnvForModule(m.id, db)); } catch { // Credential lookup failed — leave empty; terraform plan will // surface the genuine "missing var" error and the user can // fix the underlying credential. terraformEnvByModule.set(m.id, {}); } }), ); const terraformModules = deployedModules.map((m) => ({ id: m.id, terraformDir: existsSync(join(m.sourcePath, 'generated', 'terraform')) ? join(m.sourcePath, 'generated', 'terraform') : null, envVars: terraformEnvByModule.get(m.id) ?? {}, })); const healthResults = await runAllHealthChecks(db, { onProgress: options.onProgress, quiet: options.quiet, }); // Module integrity. Shallow by default: the installed tree against its // baseline, and the generated project against the installed tree. Both are // local and take milliseconds. The host plane is one SSH per system and is // reached through `module verify --deep`, not from here. const { auditModule } = await import('../../module/packaging/audit'); const integrityResults = await Promise.all(installed.map((m) => auditModule(m.id, db))); // Services-credentials: try decrypting each container service's // credential envelope. Failures (missing master key, wrong // provider shape, corrupt envelope) become BLOCKED audit findings. const allServices = await listContainerServices(); const serviceCredResults: ServiceCredentialsResult[] = await Promise.all( allServices.map(async (s) => { try { await getServiceCredentials(s.id); return { serviceId: s.serviceId, name: s.name, providerName: s.providerName, error: null, }; } catch (err) { return { serviceId: s.serviceId, name: s.name, providerName: s.providerName, error: err instanceof Error ? err.message : String(err), }; } }), ); // Secrets-decryptable: try decrypting every encrypted secret in // the DB (modules / system / capabilities). One bad master key // makes every entry fail; the audit collapses that case into a // single "master key mismatch" finding. const secretResults: SecretCheckResult[] = []; try { const masterKey = await getOrCreateMasterKey(); const tryDecrypt = (env: { encryptedValue: string; iv: string; authTag: string }): | string | null => { try { decryptSecret(env, masterKey); return null; } catch (err) { return err instanceof Error ? err.message : String(err); } }; for (const s of db.select().from(secrets).all()) { secretResults.push({ scope: 'module', subject: s.moduleId, name: s.name, error: tryDecrypt(s), }); } for (const s of db.select().from(systemSecrets).all()) { secretResults.push({ scope: 'system', subject: 'system', name: s.key, error: tryDecrypt(s), }); } for (const s of db.select().from(capabilitySecrets).all()) { // Capability secrets can be metadata-only (encrypted fields // null) — skip those, they have nothing to decrypt. if (!s.encryptedValue || !s.iv || !s.authTag) continue; secretResults.push({ scope: 'capability', subject: `capability:${s.capabilityId}`, name: s.name, error: tryDecrypt({ encryptedValue: s.encryptedValue, iv: s.iv, authTag: s.authTag, }), }); } } catch (err) { // Master key unavailable — surface as a single system-level finding. secretResults.push({ scope: 'system', subject: 'system', name: '', error: err instanceof Error ? err.message : String(err), }); } // Services-reachable: ping each container service's API in parallel // (Proxmox /version, DigitalOcean /v2/account). Reuses the same // probes the `service add` wizard runs. Services with bad // credentials get reachable=true here so we don't double-report — // the credentials audit already flagged them. const serviceReachableResults: ServiceReachableResult[] = await Promise.all( allServices.map(async (s): Promise => { try { const creds = await getServiceCredentials(s.id); let probe: { success: boolean; message?: string }; if (s.providerName === 'proxmox' && 'api_url' in creds) { probe = await testProxmoxConnection(creds); } else if (s.providerName === 'digitalocean' && 'api_token' in creds) { probe = await testDigitalOceanConnection(creds); } else { return { serviceId: s.serviceId, name: s.name, providerName: s.providerName, reachable: true, }; } return { serviceId: s.serviceId, name: s.name, providerName: s.providerName, reachable: probe.success, message: probe.success ? undefined : (probe.message ?? 'unreachable'), }; } catch { return { serviceId: s.serviceId, name: s.name, providerName: s.providerName, reachable: true, }; } }), ); // Machines-reachable and disk-space both SSH every system, so they run // concurrently here rather than paying each other's latency. The machine // probe is shared with the monitor sweep (services/machine-probe.ts) — // this block used to be a second copy of the same SSH logic, and both // copies reported the local management box as unreachable because celilo // does not hold an SSH key for itself. // // The disk probe measures every system's root filesystem — the machine // pool AND every provisioned instance, the systems celilo#1133 filled — // in the same pre-computed measurement shape the disk_space monitor // sweep consumes, so an audit finding and an alert page carry the same // numbers. const [machineReachableResults, diskUsageResults] = await Promise.all([ probeMachines(), probeDiskUsage(db), ]); const migrationsFolder = findMigrationsFolderSafe(); return { cliVersion: { installedVersion: readInstalledCliVersion(), fetcher: fetchLatestCliVersion satisfies LatestCliVersionFetcher, }, schema: { journal: migrationsFolder ? makeJournalReader(migrationsFolder) : () => null, applied: readAppliedMigrations, db, }, capabilityAbi: { modules: deployedModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, browserPin: collectBrowserPinDeps(deployedModules), terraformPlan: { modules: terraformModules, run: realTerraformPlan, }, moduleVersions: { installed: deployedModules.map((m) => ({ id: m.id, version: m.version })), fetcher: makeRegistryFetcher(registryClient), }, moduleConfigs: { modules: installedConfigs }, moduleIntegrity: { results: integrityResults }, detectWithoutConverge: { modules: deployedModules.map((m) => ({ id: m.id, state: m.state, manifest: m.manifestData as ModuleManifest, })), }, jailExemptions: { exemptions: collectJailExemptions(db) }, health: { results: healthResults }, backups: { modules: installedBackupInfo }, abandonedOperations: { records: loadAbandonedOperations(db) }, undeployedModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, errorMessage: m.errorMessage, })), }, unconfiguredModules: { modules: installed.map((m) => ({ id: m.id, state: m.state, configCount: configsByModule.get(m.id) ? Object.keys(configsByModule.get(m.id) ?? {}).length : 0, })), }, servicesCredentials: { results: serviceCredResults }, secretsDecryptable: { results: secretResults }, servicesReachable: { results: serviceReachableResults }, machinesReachable: { results: machineReachableResults }, // The only check here whose vantage point is OUTSIDE the fleet. Its // undetermined counters are read but not written from this path: an // operator-run audit is not a run of a schedule, so counting it towards // "N consecutive runs found no evidence" would misreport how long the // fleet has been unverifiable. The monitor sweep owns that (D2). publicDns: { records: loadPublicDnsRecords(db), probe: createPublicDnsProbe(loadPublicDnsProbeSettings(db)), evidence: loadPublicDnsEvidence(db), }, diskSpace: { results: diskUsageResults }, // Reads the record the poller already writes — this check never performs a // read of its own. One that did would drain the queue and eat the // acknowledgement it exists to protect (#541). transportReads: { statuses: readAllTransportStatuses(db), now: new Date(), // Six missed polls. The poller runs every five minutes, so this absorbs a // slow sweep, a restart, and a missed tick without crying wolf — while // still catching a transport that has genuinely stopped being readable. staleAfterMs: 30 * 60_000, }, trustedSources: collectFirewallReach(db), }; } const VERDICT_ICON: Record = { READY: '●', UNKNOWN: '?', DRIFT: '⚠', BLOCKED: '✗', }; const SEVERITY_ICON: Record = { unmeasured: '?', drift: '⚠', blocked: '✗', }; /** Format a `SystemAuditReport` for a human reader. */ export function formatReport(report: SystemAuditReport): string { const lines: string[] = []; lines.push(''); lines.push( `${VERDICT_ICON[report.verdict] ?? '?'} ${report.verdict} (${report.findings.length} finding${report.findings.length === 1 ? '' : 's'})`, ); lines.push(` generated at ${report.generatedAt}`); if (report.findings.length === 0) { lines.push(''); lines.push(' System is up to date. No drift detected.'); return lines.join('\n'); } // Group findings by severity, then by category. const byCategory = new Map(); for (const f of report.findings) { const key = `${f.severity}/${f.category}`; const arr = byCategory.get(key) ?? []; arr.push(f); byCategory.set(key, arr); } // Sort: blocked first, then by category name. const keys = [...byCategory.keys()].sort((a, b) => { const aBlocked = a.startsWith('blocked/'); const bBlocked = b.startsWith('blocked/'); if (aBlocked !== bBlocked) return aBlocked ? -1 : 1; return a.localeCompare(b); }); for (const key of keys) { const findings = byCategory.get(key) ?? []; const [severity, category] = key.split('/'); lines.push(''); lines.push( ` ${SEVERITY_ICON[severity]} ${severity.toUpperCase()} • ${category} (${findings.length})`, ); for (const f of findings) { lines.push(` - ${f.message}`); if (f.details) { for (const detail of f.details.split('\n')) { lines.push(` ${detail}`); } } if (f.remediation) { lines.push(` → ${f.remediation}`); } } } return lines.join('\n'); } export async function handleSystemAudit( _args: string[], flags: Record, ): Promise { const json = hasFlag(flags, 'json'); const explicitTui = hasFlag(flags, 'tui'); const noTui = hasFlag(flags, 'no-tui'); // Default to the TUI when stdout is an interactive terminal AND // the user hasn't asked for a non-interactive output mode. JSON // and --no-tui both opt out; piped/redirected stdout falls back // to the static text report so the audit stays scriptable. const tui = !json && !noTui && (explicitTui || Boolean(process.stdout.isTTY)); if (tui) { // Defer the audit to inside the TUI so progress messages render // in the alt-screen rather than leaking to the user's scrollback. const themeRaw = String(flags.theme ?? '').toLowerCase(); const theme = themeRaw === 'light' ? 'light' : themeRaw === 'dark' ? 'dark' : undefined; const { renderAuditTui } = await import('../tui/audit-tui'); try { await renderAuditTui( async (onProgress, onCategory) => { // Bracket the two long phases so the user sees something // before health checks begin emitting per-module messages, // and so the spinner caption keeps changing during the // parallel-audit phase (which doesn't emit its own). onProgress('Reading system state…'); const deps = await buildAuditDeps({ onProgress }); onProgress('Analyzing drift across categories…'); return runAudit(deps, onCategory); }, { theme }, ); return { success: true, message: '' }; } catch (err) { return { success: false, error: err instanceof Error ? err.message : String(err), }; } } let deps: Awaited>; try { deps = await buildAuditDeps({ quiet: json }); } catch (err) { return { success: false, error: `audit failed to gather state: ${err instanceof Error ? err.message : String(err)}`, }; } const report = await runAudit(deps); if (json) { return { success: true, message: JSON.stringify(report, null, 2), rawOutput: true, }; } return { success: true, message: formatReport(report), }; }