/** * `celilo system doctor` — diagnose system-level health for celilo: * 1. System prerequisites (ansible, terraform, ssh, etc.) present * and at supported versions. * 2. @celilo/* version drift between the running CLI and the * surrounding workspace (if any). * * The canonical failures this catches: * - "I just installed celilo on a fresh box but module import is * erroring with a child-process exit 127." → prereq section. * - "I edited the workspace but my global celilo is still running * an older published version." → drift section. * * Renamed from top-level `celilo doctor` (Phase 0; no alias kept) to * fit alongside `system init`, `system audit`, `system config`. See * apps/celilo/designs/PREREQ_DETECTION.md. * * Drift resolution strategy: * - The running CLI's package.json comes from a relative import — * that anchors us to whatever copy of `@celilo/cli` is actually * executing (workspace TS source or globally-installed * node_modules tree). * - For each `@celilo/*` dependency, we ask the runtime where it * resolves the package's `package.json` and read the version * there. * - If we can find a workspace root by walking up from * `process.cwd()`, we read each `packages/*\/package.json` and * flag anything where the loaded version is older than the * workspace. */ import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync, statSync } from 'node:fs'; import { createRequire } from 'node:module'; import { hostname } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { resolveBrowser } from '@celilo/capabilities'; import { defineEvents, openBus } from '@celilo/event-bus'; import cliPkg from '../../../package.json' with { type: 'json' }; import { getDbPath, getEventBusPath } from '../../config/paths'; import { getDb } from '../../db/client'; import { JAIL_OFF_REASON, type JailAvailability, type JailModeRecord, type JailPolicy, type JailPolicySource, SANDBOX_EXEC_AUTO_DEFERRED_REASON, autoJailDefers, detectJailBackend, readJailMode, resolveJailPolicy, } from '../../hooks/jail'; import type { ModuleManifest } from '../../manifest/schema'; import { HOOK_JAIL_CHECK } from '../../services/alerting/hook-jail'; import { findMonitorByTarget } from '../../services/alerting/monitors'; import { type FleetFinding, type FleetFindingStatus, checkSubscribers, runFleetChecks, } from '../../services/fleet-checks'; import { type JailExemption, collectJailExemptions } from '../../services/jail-exemptions'; import { resyncAllSubscriptions } from '../../services/module-subscriptions'; import { checkAllPrerequisites, failingPrerequisites } from '../../system/prereqs'; import type { CommandResult } from '../types'; interface CeliloPkgInfo { name: string; declaredRange: string; loadedVersion: string | null; loadedFrom: string | null; resolveError: string | null; } interface WorkspaceVersion { name: string; version: string; path: string; } const ANSI = { reset: '\x1b[0m', dim: '\x1b[2m', green: '\x1b[32m', yellow: '\x1b[33m', red: '\x1b[31m', }; /** * Discover every `@celilo/*` entry in the running CLI's package.json * dependencies and resolve where each one is actually loaded from. */ function inspectCeliloDeps(): CeliloPkgInfo[] { const deps: Record = { ...((cliPkg as { dependencies?: Record }).dependencies ?? {}), }; const celiloDeps = Object.entries(deps) .filter(([name]) => name.startsWith('@celilo/')) .sort(([a], [b]) => a.localeCompare(b)); const require = createRequire(import.meta.url); return celiloDeps.map(([name, declaredRange]) => { try { const pkgJsonPath = require.resolve(`${name}/package.json`); const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')) as { version: string }; return { name, declaredRange, loadedVersion: pkg.version, loadedFrom: dirname(pkgJsonPath), resolveError: null, }; } catch (err) { return { name, declaredRange, loadedVersion: null, loadedFrom: null, resolveError: err instanceof Error ? err.message : String(err), }; } }); } /** * Walk up from `start` until we find a `package.json` whose `workspaces` * key is non-empty. Returns the directory containing that file or null. */ function findWorkspaceRoot(start: string): string | null { let dir = resolve(start); while (true) { const candidate = join(dir, 'package.json'); if (existsSync(candidate)) { try { const pkg = JSON.parse(readFileSync(candidate, 'utf-8')) as { workspaces?: string[] | { packages?: string[] }; }; const workspaces = Array.isArray(pkg.workspaces) ? pkg.workspaces : (pkg.workspaces?.packages ?? []); if (workspaces.length > 0) return dir; } catch { /* malformed package.json — keep walking */ } } const parent = dirname(dir); if (parent === dir) return null; dir = parent; } } /** * Read every `@celilo/*` package.json in the workspace and return the * version each one declares. Used to compare against what the running * CLI actually loaded. */ function collectWorkspaceVersions(workspaceRoot: string): WorkspaceVersion[] { const out: WorkspaceVersion[] = []; // Hard-code the two glob roots used in this monorepo to avoid pulling // in a glob library. Both directories are scanned the same way: read // each immediate child, look for a package.json that names a @celilo/* // package. for (const dir of ['packages', 'apps']) { const root = join(workspaceRoot, dir); if (!existsSync(root)) continue; for (const entry of readSubdirs(root)) { const pkgJsonPath = join(root, entry, 'package.json'); if (!existsSync(pkgJsonPath)) continue; try { const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')) as { name?: string; version?: string; }; if (pkg.name?.startsWith('@celilo/') && pkg.version) { out.push({ name: pkg.name, version: pkg.version, path: join(root, entry) }); } } catch { /* skip malformed */ } } } return out; } function readSubdirs(dir: string): string[] { // node:fs readdirSync — keep stdlib, no extra deps. // Hidden dirs filtered out. try { const fs = require('node:fs') as typeof import('node:fs'); return fs .readdirSync(dir, { withFileTypes: true }) .filter((d) => d.isDirectory() && !d.name.startsWith('.')) .map((d) => d.name); } catch { return []; } } /** * Compare two semver-ish version strings. Returns -1 if a < b, 0 if * equal, 1 if a > b. Tolerates non-numeric prerelease tags by comparing * them as strings after the numeric segments. * * Exported for unit testing. */ export function compareVersions(a: string, b: string): number { const split = (v: string) => v.replace(/^[v=]+/, '').split(/[.+-]/); const aParts = split(a); const bParts = split(b); const len = Math.max(aParts.length, bParts.length); for (let i = 0; i < len; i++) { const ap = aParts[i] ?? '0'; const bp = bParts[i] ?? '0'; const an = Number(ap); const bn = Number(bp); if (!Number.isNaN(an) && !Number.isNaN(bn)) { if (an !== bn) return an < bn ? -1 : 1; } else if (ap !== bp) { return ap < bp ? -1 : 1; } } return 0; } interface DriftedDep { name: string; loadedVersion: string; workspaceVersion: string; workspacePath: string; } /** * Run a command, capture stdout/stderr, return whether it succeeded. * Used for the `bun link` calls that --fix orchestrates. */ function runCommand( cmd: string, args: string[], cwd: string, ): { ok: boolean; stdout: string; stderr: string } { const r = spawnSync(cmd, args, { cwd, encoding: 'utf-8' }); return { ok: r.status === 0, stdout: (r.stdout ?? '').trim(), stderr: (r.stderr ?? '').trim(), }; } /** * Repair drift by `bun link`-ing each drifted package from the * workspace into the running CLI's package directory. * * Two-step bun link workflow: * 1. From each workspace package dir: `bun link` registers it * globally under its package name. * 2. From the running CLI's package dir: `bun link ` * replaces the resolved copy with the symlink to the workspace. * * `bun unlink` reverses both steps if the user wants to revert. * * Only safe to run when running from a globally-installed CLI; from * a workspace TS-source invocation there's nothing to repair. */ function applyFix(drifted: DriftedDep[], cliRoot: string): string[] { const lines: string[] = []; for (const d of drifted) { lines.push(` ${d.name}: linking ${d.workspaceVersion} from ${d.workspacePath}`); const reg = runCommand('bun', ['link'], d.workspacePath); if (!reg.ok) { lines.push( ` ${ANSI.red}✗${ANSI.reset} register failed: ${reg.stderr || reg.stdout || 'no output'}`, ); continue; } const link = runCommand('bun', ['link', d.name], cliRoot); if (!link.ok) { lines.push( ` ${ANSI.red}✗${ANSI.reset} link failed: ${link.stderr || link.stdout || 'no output'}`, ); continue; } lines.push(` ${ANSI.green}✔${ANSI.reset} linked`); } return lines; } /** * Render the system-prerequisites block: every tool from the * PREREQUISITES table with its detection result, formatted into the * doctor's output column-aligned style. */ function renderPrereqSection(): { lines: string[]; failingCount: number } { const checks = checkAllPrerequisites(); const lines: string[] = []; const browserFlavor = provisionedBrowserFlavor(); lines.push('System prerequisites'); // Right-pad column for alignment. Take the longest tool name +1 // for spacing. const nameCol = Math.max(...checks.map((c) => c.name.length), 12); for (const c of checks) { if (c.present && c.meetsMinimum) { const version = c.version ? c.version : `${ANSI.dim}(version unknown)${ANSI.reset}`; // Flavour rides on the browser row because the ceiling it names is // real — a headless-only build cannot run headed — and an operator // reading doctor is the other party who benefits from that being // legible rather than discovered by a launch failure. It cannot come // from the check itself: a headless shell reports // "Google Chrome for Testing ", the same string a full build // prints, so only the descriptor knows which one is installed. const flavor = c.name === 'browser' && browserFlavor ? ` ${ANSI.dim}(${browserFlavor})${ANSI.reset}` : ''; lines.push(` ${ANSI.green}✔${ANSI.reset} ${c.name.padEnd(nameCol)} ${version}${flavor}`); } else if (c.present && !c.meetsMinimum) { // Present but below minimum (or version-parse failed with a minimum). const detail = c.version ? `${c.version} — below minimum required` : 'version unreadable'; lines.push(` ${ANSI.yellow}⚠${ANSI.reset} ${c.name.padEnd(nameCol)} ${detail}`); lines.push(` ${ANSI.dim}install: ${c.installHint}${ANSI.reset}`); } else { // Missing entirely. lines.push( ` ${ANSI.red}✗${ANSI.reset} ${c.name.padEnd(nameCol)} ${ANSI.dim}not installed${ANSI.reset}`, ); lines.push(` ${ANSI.dim}install: ${c.installHint}${ANSI.reset}`); } } return { lines, failingCount: failingPrerequisites(checks).length }; } /** * Which flavour of browser is provisioned, or null when none is (or the * descriptor cannot be read). Never throws — a doctor row is not worth * failing the whole report over. * * This reads the DESCRIPTOR while the prerequisite check RUNS the binary. * Those answer different questions and both are wanted: executing it is * what distinguishes a real install from a build directory with no binary * in it, and only the descriptor records what was installed. */ function provisionedBrowserFlavor(): string | null { try { return resolveBrowser().flavor; } catch { return null; } } export interface HookExecutionInput { availability: JailAvailability; policy: JailPolicy; /** Which source decided `policy` (D4 of hook-jail-config-surface). */ source: JailPolicySource; /** Raw `CELILO_HOOK_JAIL` value, when set; rendered on the source line. */ envValue: string | undefined; /** Raw stored `hooks.jail_policy` value, when present; rendered on the disagreement line. */ configValue: string | undefined; /** What the last real module-hook run recorded, if any. */ record: JailModeRecord | undefined; host: string; /** * Whether a `hook_jail` monitor exists. `null` when there is no celilo DB to * ask (a dev box), which also suppresses the add-the-monitor hint. */ monitored: boolean | null; /** * Modules running under a recorded policy weaker than the system's * (per-module-jail-policy task 3.2), from the shared collector. Empty is * the normal state and renders nothing. */ exemptions: readonly JailExemption[]; } /** * The hook-execution section (hook-process-boundary task 4.6). * * Reports design D8's three states the way the design assigns them: jailed is * a ✔ and nothing more; unjailed-and-always-was is reported with the reason in * one actionable sentence and is a warning, not an event; a host that USED to * jail and has stopped is a failure, and the self-monitor (`celilo monitor add * hook_jail`) is what turns it into an alert. * * The headline is the LIVE answer — the probe runs bubblewrap the way a real * spawn would — while the recorded mode says what the last hook actually did. * They disagree exactly when the host changed since the last hook ran, which * is worth seeing, not reconciling away. * * Pure so every branch is testable on a host with no jail (Rule 10.4); the * caller injects what the machine says. */ export function renderHookExecutionSection(input: HookExecutionInput): { lines: string[]; failCount: number; warnCount: number; } { const { availability, policy, record, host, monitored } = input; const lines: string[] = ['Hook execution']; let failCount = 0; let warnCount = 0; // Which source decided the policy (D4 of hook-jail-config-surface). When // env and config both hold values and disagree, name the loser too: a // control whose two sources disagree and whose rendering hides one of them // is the exact failure this change fixes, one level down. if (input.source === 'env' && input.envValue !== undefined) { lines.push( ` ${ANSI.dim}policy from CELILO_HOOK_JAIL=${input.envValue} (environment override)${ANSI.reset}`, ); } else if (input.source === 'config') { lines.push(` ${ANSI.dim}policy from system config (hooks.jail_policy)${ANSI.reset}`); } else { // Read the default off `policy` rather than naming it. The literal that // stood here said `auto` and stayed saying it after ce-rez7 flipped the // default to `off`, so the doctor named the one value not in play. lines.push( ` ${ANSI.dim}policy default (${policy}) — set hooks.jail_policy or CELILO_HOOK_JAIL to change${ANSI.reset}`, ); } if ( input.source === 'env' && input.configValue !== undefined && input.envValue !== input.configValue ) { lines.push( ` ${ANSI.dim}system config holds '${input.configValue}' but CELILO_HOOK_JAIL='${input.envValue}' overrides it${ANSI.reset}`, ); } // ce-29z: auto defers on sandbox-exec, so the doctor must not report a jail // the executor will not build. One predicate, two consumers. const autoDeferred = policy === 'auto' && autoJailDefers(availability); const liveJailed = policy !== 'off' && availability.backend !== 'none' && !autoDeferred; if (policy === 'required' && availability.backend === 'none') { lines.push( ` ${ANSI.red}✗${ANSI.reset} the hook jail policy is 'required' and no jail is available — every module hook on this host fails rather than run unjailed`, ); lines.push(` ${ANSI.dim}${availability.reason ?? 'no reason was recorded'}${ANSI.reset}`); failCount++; } else if (policy === 'off') { lines.push(` ${ANSI.yellow}⚠${ANSI.reset} hooks run unjailed — ${JAIL_OFF_REASON}`); warnCount++; } else if (availability.backend === 'none') { lines.push( ` ${ANSI.yellow}⚠${ANSI.reset} hooks run unjailed — ${availability.reason ?? 'no reason was recorded'}`, ); warnCount++; } else if (autoDeferred) { lines.push( ` ${ANSI.yellow}⚠${ANSI.reset} hooks run unjailed — ${SANDBOX_EXEC_AUTO_DEFERRED_REASON}`, ); warnCount++; } else { lines.push(` ${ANSI.green}✔${ANSI.reset} hooks run jailed (${availability.backend})`); } // Per-module exemptions (per-module-jail-policy task 3.2). Loud and // countable is the whole design: the count AND the names, every time, and // nothing at all when there are none. if (input.exemptions.length > 0) { const n = input.exemptions.length; const named = input.exemptions.map((e) => `${e.moduleId} (${e.policy})`).join(', '); lines.push( ` ${ANSI.yellow}⚠${ANSI.reset} ${n} module${n === 1 ? '' : 's'} run${n === 1 ? 's' : ''} under a weaker jail policy than the system's: ${named}`, ); warnCount++; } const regressed = record && record.host === host && record.mode === 'unjailed' && record.lastJailed; if (regressed && record.lastJailed) { lines.push( ` ${ANSI.red}✗${ANSI.reset} this host ran hooks jailed (${record.lastJailed.backend}) until ${record.lastJailed.recordedAt} and has stopped`, ); if (record.reason) lines.push(` ${ANSI.dim}${record.reason}${ANSI.reset}`); failCount++; if (monitored === false) { lines.push( ` ${ANSI.dim}→ nothing alerts on this — celilo monitor add ${HOOK_JAIL_CHECK}${ANSI.reset}`, ); } } else if (!record) { lines.push( ` ${ANSI.dim}no execution mode recorded yet — the first real module-hook run writes it${ANSI.reset}`, ); } else if (record.host !== host) { lines.push( ` ${ANSI.dim}last recorded mode belongs to host "${record.host}" (this data directory moved) — the next hook run re-records${ANSI.reset}`, ); } else { const backend = record.mode === 'jailed' ? ` (${record.backend})` : ''; lines.push( ` ${ANSI.dim}last hook ran ${record.mode}${backend} at ${record.recordedAt}${ANSI.reset}`, ); } // The moment this hint pays for itself is BEFORE a regression: the monitor // has to exist for the transition to page anyone. if (liveJailed && monitored === false) { lines.push( ` ${ANSI.dim}→ a host that stops jailing alerts only if monitored: celilo monitor add ${HOOK_JAIL_CHECK}${ANSI.reset}`, ); } return { lines, failCount, warnCount }; } /** Where `assembleHookExecutionSection` gets what the machine says. Every * reader is a thunk so any of their throws (a bad stored row, a corrupt * record) land in the one catch that renders this section. */ export interface HookExecutionIO { envValue: () => string | undefined; configValue: () => string | undefined; availability: () => JailAvailability; record: () => JailModeRecord | undefined; host: () => string; monitored: () => boolean | null; /** Modules exempted from the system's jail policy (per-module-jail-policy task 3.2). */ exemptions: () => JailExemption[]; } /** * Resolve the effective jail policy from its two sources and render the * Hook execution section (hook-jail-config-surface D2/D4). * * The single catch is the section's one error path: a typo'd env value or a * bad stored row renders as a failing check rather than crashing the doctor, * because the same throw is about to fail every hook invocation too. */ export function assembleHookExecutionSection(io: HookExecutionIO): { lines: string[]; failCount: number; warnCount: number; } { try { const envValue = io.envValue(); const configValue = io.configValue(); // The module step of the four-step precedence stays out of the headline // resolution — the headline is the policy the EXECUTOR would use for a // module with no row of its own. Per-module rows are surfaced separately // as exemptions (per-module-jail-policy task 3.2), which is the shape the // operator acts on. const resolved = resolveJailPolicy(envValue, undefined, configValue); return renderHookExecutionSection({ availability: io.availability(), policy: resolved.policy, source: resolved.source, envValue, configValue, record: io.record(), host: io.host(), monitored: io.monitored(), exemptions: io.exemptions(), }); } catch (error) { return { lines: [ 'Hook execution', ` ${ANSI.red}✗${ANSI.reset} ${error instanceof Error ? error.message : String(error)}`, ], failCount: 1, warnCount: 0, }; } } /** * The stored `hooks.jail_policy` value, or undefined when there is no row or * no celilo DB to ask (a dev box). Same row-read pattern as * `ipam/auto-allocator.ts`; the drizzle table is `systemConfig` in * `db/schema.ts`. */ function readStoredJailPolicy(): string | undefined { if (!existsSync(getDbPath())) return undefined; const row = getDb() .$client.prepare('SELECT value FROM system_config WHERE key = ?') .get('hooks.jail_policy') as { value: string } | undefined; return row?.value; } /** * mtime (ms) of the installed dispatcher code (`@celilo/event-bus` * package.json). The fleet dispatcher check compares this against the * running dispatcher's start time to spot a process on stale code. Null * when the package can't be located (the staleness aspect is then skipped). */ function installedEventBusMtime(): number | null { try { const require = createRequire(import.meta.url); return statSync(require.resolve('@celilo/event-bus/package.json')).mtimeMs; } catch { return null; } } const FLEET_GLYPH: Record = { ok: `${ANSI.green}✔${ANSI.reset}`, warn: `${ANSI.yellow}⚠${ANSI.reset}`, fail: `${ANSI.red}✗${ANSI.reset}`, }; function renderFleetFinding(f: FleetFinding): string[] { const lines = [` ${FLEET_GLYPH[f.status]} ${f.title} ${ANSI.dim}— ${f.summary}${ANSI.reset}`]; for (const d of f.detail) lines.push(` ${ANSI.dim}${d}${ANSI.reset}`); if (f.remediation && f.status !== 'ok') lines.push(` ${ANSI.dim}→ ${f.remediation}${ANSI.reset}`); return lines; } /** * The fleet-runtime section: dispatcher / subscribers / capability-chain * drift, read from the celilo DB + event bus. State-aware, so it's skipped * cleanly on a fresh dev box with no DB unless `--fleet` forces it. `--fix` * runs only the auto-fixable checks (today: subscribers resync). */ /** * Aspect coverage, measured against every entitled system (celilo#902 design D7). * * Behind `--deep` and never in a default pass, because it SSHes to every * system an approved aspect covers. That gating is also what keeps it honest: * there is no "we have no data yet" state to explain away — the check either * ran and measured, or it did not run. * * `--fix` converges only the hosts measured as MISSING. An `unknown` host is * never converged: celilo does not know what state it is in, so there is * nothing to converge toward, and re-running the role anyway would be inventing * the verdict this check exists to avoid. */ async function renderAspectCoverage(opts: { fix: boolean }): Promise<{ lines: string[]; failCount: number; warnCount: number; }> { const { verifyAspectCoverage } = await import('../../services/aspect-runner'); const { getDb } = await import('../../db/client'); const lines: string[] = ['Aspect coverage']; const findings = await verifyAspectCoverage({ db: getDb() }); if (findings.length === 0) { lines.push(` ${ANSI.dim}no approved base-module aspects to verify${ANSI.reset}`); return { lines, failCount: 0, warnCount: 0 }; } const missing = findings.filter((f) => f.state === 'missing'); const unknown = findings.filter((f) => f.state === 'unknown' || f.state === 'unreachable'); const applied = findings.filter((f) => f.state === 'applied'); for (const f of applied) { lines.push(` ${ANSI.green}✓${ANSI.reset} ${f.hostname} has ${f.providerModuleId}/${f.role}`); } for (const f of missing) { lines.push( ` ${ANSI.red}✗${ANSI.reset} ${f.hostname} is MISSING ${f.providerModuleId}/${f.role}`, ); if (f.detail) lines.push(` ${ANSI.dim}${f.detail}${ANSI.reset}`); } for (const f of unknown) { lines.push( ` ${ANSI.yellow}?${ANSI.reset} ${f.hostname} — ${f.providerModuleId}/${f.role} NOT MEASURED`, ); if (f.detail) lines.push(` ${ANSI.dim}${f.detail}${ANSI.reset}`); } if (opts.fix && missing.length > 0) { // Converged per PROVIDER with `onlyHostnames`, not through // `reconcileAspectsForSystems`. Entitlement was already established by // `planAspectFanOut` when the finding was measured, and consent was already // confirmed 'approved' — re-deriving either from the zone would only add a // second path that can disagree with the first. const { runAspectFanOut } = await import('../../services/aspect-runner'); const { modules } = await import('../../db/schema'); const { eq } = await import('drizzle-orm'); const db = getDb(); const byProvider = new Map(); for (const f of missing) { byProvider.set(f.providerModuleId, [ ...(byProvider.get(f.providerModuleId) ?? []), f.hostname, ]); } lines.push(''); lines.push(` Converging ${missing.length} host/aspect pair(s) measured as missing:`); for (const [providerModuleId, hostnames] of byProvider) { const row = db.select().from(modules).where(eq(modules.id, providerModuleId)).get(); const aspect = (row?.manifestData as ModuleManifest | null)?.base_module_aspect; if (!row || !aspect) continue; const result = await runAspectFanOut({ moduleId: providerModuleId, aspect, moduleSourcePath: row.sourcePath, options: { trigger: 'on_new_system_in_zone', onlyHostnames: hostnames }, db, }); const mark = result.success ? `${ANSI.green}✓${ANSI.reset}` : `${ANSI.red}✗${ANSI.reset}`; lines.push( ` ${mark} ${providerModuleId} → ${hostnames.join(', ')}${result.success ? '' : `: ${result.error ?? 'unknown error'}`}`, ); } // Re-MEASURED on the next --deep run rather than asserted from this run's // exit status. "The command exited 0" is the verdict, not the contract. lines.push( ` ${ANSI.dim}Re-run \`celilo system doctor --deep\` to confirm against the hosts.${ANSI.reset}`, ); return { lines, failCount: 0, warnCount: unknown.length }; } if (missing.length > 0) { lines.push( ` ${ANSI.dim}Run \`celilo system doctor --deep --fix\` to apply the missing aspects.${ANSI.reset}`, ); } return { lines, failCount: missing.length, warnCount: unknown.length }; } /** * The fleet-section line for module integrity, one row per module. * * `module verify` is the detailed surface and `system audit` is the machine * one; doctor gets the summary, because doctor is where an operator looks * first. Same implementation behind all three. */ async function renderModuleIntegrity(opts: { db: ReturnType; deep: boolean; }): Promise<{ lines: string[]; failCount: number; warnCount: number }> { const { auditModule } = await import('../../module/packaging/audit'); const { auditModuleIntegrity } = await import('../../services/audit/module-integrity'); const { modules: modulesTable } = await import('../../db/schema'); const installed = opts.db.select().from(modulesTable).all(); if (installed.length === 0) return { lines: [], failCount: 0, warnCount: 0 }; const results = await Promise.all( installed.map((m) => auditModule(m.id, opts.db, { deep: opts.deep })), ); const findings = auditModuleIntegrity({ results }); const drifted = findings.filter((f) => f.severity === 'drift'); const unmeasured = findings.filter((f) => f.severity === 'unmeasured'); if (findings.length === 0) { const scope = opts.deep ? 'including what is running on their hosts' : 'installed and generated'; return { lines: [ ` ${ANSI.green}✔${ANSI.reset} Module integrity ${ANSI.dim}— ${installed.length} module(s) match their recorded version (${scope})${ANSI.reset}`, ], failCount: 0, warnCount: 0, }; } const lines: string[] = [ ` ${drifted.length > 0 ? `${ANSI.red}✗${ANSI.reset}` : `${ANSI.yellow}?${ANSI.reset}`} Module integrity ${ANSI.dim}— ${drifted.length} drifted, ${unmeasured.length} unmeasured, of ${installed.length} module(s)${ANSI.reset}`, ]; // Every finding, never just the first. celilo#951 printed `drift[0]` of 19, // so a real finding would have arrived at position 19 and never been seen. for (const f of findings) { lines.push(` ${ANSI.dim}${f.message}${ANSI.reset}`); if (f.remediation) lines.push(` ${ANSI.dim}→ ${f.remediation}${ANSI.reset}`); } if (!opts.deep) { lines.push( ` ${ANSI.dim}Run \`celilo system doctor --deep\` to also ask each host what it is running.${ANSI.reset}`, ); } return { lines, failCount: drifted.length, warnCount: unmeasured.length }; } async function renderFleetSection(opts: { forced: boolean; fix: boolean; deep: boolean; }): Promise<{ lines: string[]; failCount: number; warnCount: number; }> { const lines: string[] = ['Fleet runtime']; const dbExists = existsSync(getDbPath()); if (!dbExists) { if (opts.forced) { lines.push( ` ${ANSI.dim}skipped — no celilo database at ${getDbPath()} (not a management plane)${ANSI.reset}`, ); return { lines, failCount: 0, warnCount: 0 }; } // Dev box, no --fleet: don't render the section at all. return { lines: [], failCount: 0, warnCount: 0 }; } const bus = openBus({ dbPath: getEventBusPath(), events: defineEvents({}) }); try { const db = getDb(); let findings = await runFleetChecks(bus, db, { installedCodeMtimeMs: installedEventBusMtime(), }); if (opts.fix) { const subscribers = findings.find((f) => f.id === 'subscribers'); if (subscribers?.autoFixable && subscribers.status !== 'ok') { const r = resyncAllSubscriptions(); lines.push( ` ${ANSI.dim}--fix: re-registered ${r.registered} subscription(s) from ${r.modules} module(s).${ANSI.reset}`, ); // Re-evaluate the subscribers finding so the section shows the healed state. const healed = checkSubscribers(bus, db); findings = findings.map((f) => (f.id === 'subscribers' ? healed : f)); } } let failCount = 0; let warnCount = 0; for (const f of findings) { lines.push(...renderFleetFinding(f)); if (f.status === 'fail') failCount++; else if (f.status === 'warn') warnCount++; } // Module integrity: is the code on each box the code celilo thinks it is? // Shallow here by design — the installed tree against its baseline and the // generated project against the installed tree are both local and take // milliseconds. The host plane is one SSH per system and is reached with // `--deep`, alongside aspect coverage, for the same reason // (openspec/changes/module-integrity-rigor, D8). const integrity = await renderModuleIntegrity({ db, deep: opts.deep }); lines.push(...integrity.lines); failCount += integrity.failCount; warnCount += integrity.warnCount; return { lines, failCount, warnCount }; } finally { bus.close(); } } export async function handleSystemDoctor( _args: string[], flags: Record, ): Promise { const lines: string[] = []; const cliVersion = (cliPkg as { version: string }).version; const cliName = (cliPkg as { name: string }).name; // Where is *this* file loaded from? Anchors the "running from" line. const cliRoot = resolve(dirname(new URL(import.meta.url).pathname), '../../..'); lines.push(`${cliName} ${cliVersion}`); lines.push(`${ANSI.dim}running from ${cliRoot}${ANSI.reset}`); lines.push(''); // System prerequisites first — they're the most common reason a // fresh management box can't run modules. const prereqResult = renderPrereqSection(); lines.push(...prereqResult.lines); lines.push(''); // Hook execution mode (hook-process-boundary task 4.6; sources per D4 of // hook-jail-config-surface). The policy read throws on a typo'd // CELILO_HOOK_JAIL or a bad stored row — report that rather than crash the // doctor, since the same throw is about to fail every hook invocation too. const hookExec = assembleHookExecutionSection({ envValue: () => process.env.CELILO_HOOK_JAIL, configValue: () => readStoredJailPolicy(), availability: () => detectJailBackend(), record: () => readJailMode(), host: () => hostname(), monitored: () => existsSync(getDbPath()) ? findMonitorByTarget(getDb(), HOOK_JAIL_CHECK) !== undefined : null, exemptions: () => (existsSync(getDbPath()) ? collectJailExemptions(getDb()) : []), }); lines.push(...hookExec.lines); lines.push(''); const workspaceRoot = findWorkspaceRoot(process.cwd()); const workspaceVersions = workspaceRoot ? collectWorkspaceVersions(workspaceRoot) : []; const workspaceMap = new Map(workspaceVersions.map((w) => [w.name, w])); if (workspaceRoot) { lines.push(`workspace: ${workspaceRoot}`); } else { lines.push(`${ANSI.dim}no workspace detected from ${process.cwd()}${ANSI.reset}`); } lines.push(''); const deps = inspectCeliloDeps(); // Track whether anything is amiss so we can summarize and exit non-zero. let driftCount = 0; let unresolvedCount = 0; const drifted: DriftedDep[] = []; // Compute column widths for a clean table. const nameCol = Math.max(...deps.map((d) => d.name.length), 12); const declCol = Math.max(...deps.map((d) => d.declaredRange.length), 8); const loadedCol = Math.max(...deps.map((d) => (d.loadedVersion ?? '?').length), 8); lines.push( ` ${'package'.padEnd(nameCol)} ${'declares'.padEnd(declCol)} ${'loaded'.padEnd(loadedCol)} notes`, ); lines.push(` ${'-'.repeat(nameCol)} ${'-'.repeat(declCol)} ${'-'.repeat(loadedCol)} -----`); for (const dep of deps) { const loaded = dep.loadedVersion ?? '?'; const notes: string[] = []; let glyph = `${ANSI.green}✔${ANSI.reset}`; if (dep.resolveError) { glyph = `${ANSI.red}✗${ANSI.reset}`; notes.push(`unresolved: ${dep.resolveError.split('\n')[0]}`); unresolvedCount++; } else if (dep.loadedVersion) { const ws = workspaceMap.get(dep.name); if (ws) { const cmp = compareVersions(dep.loadedVersion, ws.version); if (cmp < 0) { glyph = `${ANSI.yellow}⚠${ANSI.reset}`; notes.push(`workspace has ${ws.version} — running CLI is behind`); driftCount++; drifted.push({ name: dep.name, loadedVersion: dep.loadedVersion, workspaceVersion: ws.version, workspacePath: ws.path, }); } else if (cmp > 0) { notes.push(`workspace has ${ws.version} (older — unpublished bump?)`); } } if (dep.loadedFrom) { const shortPath = dep.loadedFrom.replace(process.env.HOME ?? '', '~'); notes.push(`from ${shortPath}`); } } lines.push( `${glyph} ${dep.name.padEnd(nameCol)} ${dep.declaredRange.padEnd(declCol)} ${loaded.padEnd(loadedCol)} ${ANSI.dim}${notes.join('; ')}${ANSI.reset}`, ); } // Workspace packages that the CLI doesn't depend on — surface them so // the operator sees the full set of @celilo/* in play. const declaredNames = new Set(deps.map((d) => d.name)); const extras = workspaceVersions.filter((w) => !declaredNames.has(w.name)); if (extras.length > 0) { lines.push(''); lines.push(`${ANSI.dim}other workspace packages (not depended on by this CLI):${ANSI.reset}`); for (const w of extras) { lines.push(` ${w.name} ${w.version} ${ANSI.dim}${w.path}${ANSI.reset}`); } } lines.push(''); const fix = flags.fix === true; // Drift repair (bun link). Does NOT early-return — the fleet section // still runs on a --fix invocation so a single `--fix` heals both. if (fix && drifted.length > 0) { lines.push(`Repairing ${drifted.length} drifted package(s) with \`bun link\`:`); lines.push(...applyFix(drifted, cliRoot)); lines.push(`${ANSI.dim}\`bun unlink\` from each workspace dir reverses.${ANSI.reset}`); lines.push(''); // Linked from the workspace now — no longer drift for the summary. driftCount = 0; drifted.length = 0; } else if (fix && drifted.length === 0) { lines.push(`${ANSI.dim}--fix: no drifted packages to repair.${ANSI.reset}`); lines.push(''); } else if (drifted.length > 0) { lines.push( `${ANSI.dim}Run \`celilo system doctor --fix\` to bun-link drifted packages from the workspace.${ANSI.reset}`, ); lines.push(''); } // Fleet-runtime section (state-aware; only renders on a management // plane with a celilo DB, or when --fleet forces it). const fleet = await renderFleetSection({ forced: flags.fleet === true, fix, deep: flags.deep === true, }); if (fleet.lines.length > 0) { lines.push(...fleet.lines); lines.push(''); } // Aspect coverage. `--deep` only: it reaches out to every entitled system // over SSH, which a default doctor pass must not do. let aspectFailCount = 0; let aspectWarnCount = 0; if (flags.deep === true) { const coverage = await renderAspectCoverage({ fix }); lines.push(...coverage.lines); lines.push(''); aspectFailCount = coverage.failCount; aspectWarnCount = coverage.warnCount; } // Unified summary: anything that fails the run, with a per-cause count. const problems: string[] = []; if (prereqResult.failingCount > 0) { problems.push(`${prereqResult.failingCount} prerequisite(s) missing/below-minimum`); } if (driftCount > 0) problems.push(`${driftCount} package(s) behind workspace`); if (unresolvedCount > 0) problems.push(`${unresolvedCount} unresolved`); if (hookExec.failCount > 0) { problems.push(`${hookExec.failCount} hook-execution problem(s)`); } if (fleet.failCount > 0) problems.push(`${fleet.failCount} fleet check(s) failing`); if (aspectFailCount > 0) problems.push(`${aspectFailCount} system(s) missing a fleet aspect`); if (problems.length > 0) { return { success: false, error: `Issues detected: ${problems.join(', ')}`, details: lines.join('\n'), }; } // Fleet warnings don't fail the run, but they shouldn't read as a clean bill. const warnTotal = fleet.warnCount + aspectWarnCount + hookExec.warnCount; if (warnTotal > 0) { lines.push(`${ANSI.yellow}OK with warnings${ANSI.reset} — ${warnTotal} warning(s); see above`); } else { lines.push(`${ANSI.green}OK${ANSI.reset} — no issues detected`); } return { success: true, message: lines.join('\n'), rawOutput: true, }; }