import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
import { eq } from 'drizzle-orm';
import type { DbClient } from '../db/client';
import {
  buildBusHookRuns,
  capabilities as capabilitiesTable,
  dnsInternalRecords,
  moduleConfigs,
  moduleSystems,
  modules,
  systemConfig,
} from '../db/schema';
import type { ModuleManifest } from '../manifest/schema';
import { setupTestDatabaseAt } from '../test-utils/database';
import { resetTestDbPath } from '../test-utils/db-path';
import { ensureInboundSubscriber, ensureSweepSubscriber } from './alerting/monitors';
import { ensureBackupSweepSubscriber } from './backup-sweep';
import { getDaemonUnitPath } from './events-daemon';
import {
  type HostLivenessInputs,
  checkBuildBusPublishing,
  checkCapabilityProviders,
  checkControlPlaneNetwork,
  checkDispatcher,
  checkHostLiveness,
  checkSchemaDrift,
  checkServiceDns,
  checkSubscribers,
  describeCapabilityProblem,
  findBrokenCapabilityDerivations,
  runFleetChecks,
} from './fleet-checks';
import { ensureOperationsSweepSubscriber } from './module-operations';

const MINUTE = 60_000;

/** Seed a dispatcher heartbeat row directly (bypasses the running loop). */
function seedHeartbeat(
  bus: Bus,
  opts: { startedAt: number; lastHeartbeat: number; pid?: number; version?: string },
): void {
  bus.db.run(
    'INSERT INTO dispatcher_heartbeat (dispatcher_id, last_heartbeat, started_at, pid, version) VALUES (?, ?, ?, ?, ?)',
    ['d1', opts.lastHeartbeat, opts.startedAt, opts.pid ?? 4242, opts.version ?? '0.1.0'],
  );
}

/** Abandon `count` deliveries to one subscriber, oldest first. */
function seedFailed(bus: Bus, count: number): void {
  const sub = bus.subscribe({ name: 'namecheap.ddns', pattern: 'ddns.refresh', handler: 'echo' });
  for (let i = 0; i < count; i++) {
    const event = bus.emitRaw('ddns.refresh', { n: i });
    bus.markFailed(
      { eventId: event.id, subscriberId: sub.id },
      new Error('handler timed out after 30000ms'),
      { abandoned: true },
    );
  }
}

/** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
  const path = getDaemonUnitPath('linux', home, scope, systemRoot);
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, '[Unit]\nDescription=fake\n');
}

const baseManifest = (overrides: Partial<ModuleManifest> = {}): ModuleManifest => ({
  celilo_contract: '1.0',
  id: 'mod',
  name: 'Mod',
  version: '1.0.0',
  requires: { capabilities: [] },
  provides: { capabilities: [] },
  variables: { owns: [], imports: [] },
  ...overrides,
});

describe('checkDispatcher', () => {
  let dir: string;
  let dbPath: string;
  let home: string;
  let bus: Bus;
  // Use real wall-clock: bus.health() reads Date.now() internally, so a
  // synthetic past `now` would make every seeded heartbeat look stale.
  let now: number;

  beforeEach(() => {
    dir = mkdtempSync(join(tmpdir(), 'fleet-disp-'));
    dbPath = join(dir, 'events.db');
    home = join(dir, 'home');
    now = Date.now();
    process.env.EVENT_BUS_DB = dbPath;
    bus = openBus({ dbPath, events: defineEvents({}) });
  });
  afterEach(() => {
    bus.close();
    delete process.env.EVENT_BUS_DB;
    try {
      rmSync(dir, { recursive: true, force: true });
    } catch {
      /* ignore */
    }
  });

  it('fails when no dispatcher has ever heartbeated', () => {
    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    expect(f.status).toBe('fail');
    expect(f.summary).toContain('no live event dispatcher');
    expect(f.remediation).toContain('enable --now celilo-events.service');
  });

  // celilo#1373 — the unit is installed and launchd keeps respawning it, but
  // it dies before it can heartbeat (no PATH → the wrapper can't find
  // bun). The finding must NAME the crash loop, not tell the operator to
  // "start the dispatcher".
  it('names a crash-looping launchd unit when no dispatcher is live (darwin)', () => {
    installFakeUnit(home);
    const f = checkDispatcher(bus, {
      now,
      home,
      platform: 'darwin',
      launchdProbe: () => ({ pid: null, lastExitStatus: 256 }),
    });
    expect(f.status).toBe('fail');
    expect(f.detail.join(' ')).toContain('crash-looping');
    expect(f.detail.join(' ')).toContain('exit code 1');
    expect(f.remediation).toContain('celilo events install-daemon');
  });

  it('does not claim a crash loop when launchd runs the unit or cannot answer', () => {
    installFakeUnit(home);
    for (const probe of [
      () => ({ pid: 4242, lastExitStatus: 256 }), // running despite an old failure
      () => null, // launchd didn't answer — ignorance, not evidence
    ]) {
      const f = checkDispatcher(bus, {
        now,
        home,
        platform: 'darwin',
        launchdProbe: probe,
      });
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).not.toContain('crash-looping');
    }
  });

  it('passes when running, supervised, current, and not behind on timer ticks', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
    bus.emitRaw('timer.tick.15m', { interval: '15m' });

    const f = checkDispatcher(bus, {
      now: now,
      home,
      platform: 'linux',
      installedCodeMtimeMs: now - 2 * MINUTE, // code installed BEFORE the dispatcher started
    });
    expect(f.status).toBe('ok');
  });

  it('warns when the dispatcher is an orphan (no supervisor unit)', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    // no unit file written
    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    expect(f.status).toBe('warn');
    expect(f.detail.join(' ')).toContain('not under a supervisor');
  });

  // #610 — celilo-mgr had a dead system unit and a live user-scope unit of the
  // SAME name. The old file-exists test called that "supervised".
  it('fails when the running dispatcher is not the pid any installed unit supervises', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 588704 });
    installFakeUnit(home);
    const f = checkDispatcher(bus, {
      now: now,
      home,
      platform: 'linux',
      unitMainPid: () => 3639051, // systemd supervises a different process
    });
    expect(f.status).toBe('fail');
    expect(f.detail.join(' ')).toContain('not the process any installed unit supervises');
  });

  it('fails when both a user-scope and a system-scope unit are installed', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
    installFakeUnit(home);
    installFakeUnit(home, 'system', dir);
    const f = checkDispatcher(bus, {
      now: now,
      home,
      systemRoot: dir,
      platform: 'linux',
      unitMainPid: () => 4242,
    });
    expect(f.status).toBe('fail');
    expect(f.detail.join(' ')).toContain('same unit name, different services');
  });

  // Null is ignorance, not evidence: a systemd probe that fails must not
  // manufacture an orphan report.
  it('makes no supervision claim when the unit pid cannot be determined', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
    installFakeUnit(home);
    const f = checkDispatcher(bus, {
      now: now,
      home,
      platform: 'linux',
      installedCodeMtimeMs: now - 2 * MINUTE,
      unitMainPid: () => null,
    });
    expect(f.status).toBe('ok');
  });

  it('warns when the dispatcher started before the installed code (stale)', () => {
    seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    const f = checkDispatcher(bus, {
      now: now,
      home,
      platform: 'linux',
      installedCodeMtimeMs: now - 5 * MINUTE, // code newer than the running dispatcher
    });
    expect(f.status).toBe('warn');
    expect(f.detail.join(' ')).toContain('stale code');
  });

  it('warns when a timer subscriber exists but no tick was ever emitted', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
    // no tick emitted
    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    expect(f.status).toBe('warn');
    expect(f.detail.join(' ')).toContain('no such tick has ever been emitted');
  });

  it('warns when the newest timer tick is older than the window', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    bus.subscribe({ name: 'namecheap.ddns', pattern: 'timer.tick.15m', handler: 'echo' });
    bus.emitRaw('timer.tick.15m', { interval: '15m' });
    // tick exists but is ancient relative to now (emitRaw stamps Date.now()).
    bus.db.run("UPDATE events SET emitted_at = ? WHERE type = 'timer.tick.15m'", [
      now - 30 * MINUTE,
    ]);

    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    expect(f.status).toBe('warn');
    expect(f.detail.join(' ')).toContain('not emitting on schedule');
  });

  // celilo#623 — the old check read `failedDeliveries({ limit: 50 }).length`,
  // so on celilo-mgr it printed a literal `50` that meant "at least 50" and
  // read as an exact count. 137 > any limit anyone would pick.
  it('reports the TRUE total of failed deliveries, not the read limit', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    seedFailed(bus, 137);

    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    expect(f.status).toBe('warn');
    expect(f.detail.join(' ')).toContain('137 failed/abandoned delivery(ies) total');
    expect(f.detail.join(' ')).not.toContain('50 failed');
  });

  // The stored error is double-wrapped for every row written before the
  // serializeError fix; those live 90 days, so doctor must unwrap them.
  it('renders the sample error text, not {"message":"[object Object]"}', () => {
    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
    installFakeUnit(home);
    seedFailed(bus, 1);
    bus.db.run('UPDATE deliveries SET last_error = ?', [
      JSON.stringify({
        message: '[object Object]',
        value: { message: 'handler exited with code 1' },
      }),
    ]);

    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
    const detail = f.detail.join(' ');
    expect(detail).toContain('handler exited with code 1');
    expect(detail).not.toContain('[object Object]');
  });
});

describe('checkSubscribers + checkCapabilityProviders', () => {
  let dir: string;
  let dbPath: string;
  let busPath: string;
  let db: DbClient;
  let bus: Bus;

  beforeEach(async () => {
    dir = mkdtempSync(join(tmpdir(), 'fleet-subs-'));
    dbPath = join(dir, 'celilo.db');
    busPath = join(dir, 'events.db');
    process.env.CELILO_DB_PATH = dbPath;
    process.env.EVENT_BUS_DB = busPath;
    db = await setupTestDatabaseAt(dbPath);
    bus = openBus({ dbPath: busPath, events: defineEvents({}) });
  });
  afterEach(() => {
    bus.close();
    db.$client.close();
    resetTestDbPath();
    delete process.env.EVENT_BUS_DB;
    try {
      rmSync(dir, { recursive: true, force: true });
    } catch {
      /* ignore */
    }
  });

  function insertModule(id: string, manifest: ModuleManifest, state = 'VERIFIED' as const): void {
    db.insert(modules)
      .values({
        id,
        name: manifest.name,
        version: manifest.version,
        state,
        manifestData: manifest as unknown as Record<string, unknown>,
        sourcePath: `/src/${id}`,
      })
      .run();
  }

  describe('checkBuildBusPublishing', () => {
    const now = Date.now();

    /** A module whose installed manifest really declares on_upstream_publish. */
    function insertHookModule(id: string): void {
      const src = join(dir, `${id}-src`);
      mkdirSync(src, { recursive: true });
      writeFileSync(
        join(src, 'manifest.yml'),
        [
          'celilo_contract: "1.0"',
          `id: ${id}`,
          `name: ${id}`,
          'version: 1.0.0',
          'requires: { capabilities: [] }',
          'provides: { capabilities: [] }',
          'variables: { owns: [], imports: [] }',
          'hooks:',
          '  on_upstream_publish:',
          '    - name: self-update',
          '      match: { registry: npm, tag: latest, package_pattern: "@celilo/cli" }',
          '      script: ./scripts/update.sh',
        ].join('\n'),
      );
      insertModule(id, baseManifest({ id, name: id }), 'VERIFIED');
      // insertModule pins sourcePath to /src/<id>; point it at the real manifest.
      db.update(modules).set({ sourcePath: src }).where(eq(modules.id, id)).run();
    }

    function seedRun(overrides: Partial<typeof buildBusHookRuns.$inferInsert> = {}): void {
      db.insert(buildBusHookRuns)
        .values({
          eventId: 'evt-1',
          packageName: '@celilo/cli',
          packageVersion: '2.2.1',
          tag: 'latest',
          moduleId: 'celilo-mgmt',
          hookName: 'self-update',
          scriptPath: '/src/celilo-mgmt/scripts/update.sh',
          exitCode: 0,
          timedOut: false,
          durationMs: 1200,
          // Default to the freshest time, not the describe-time `now`:
          // emitRaw stamps the bus event with its own Date.now(), and a
          // run stamped even 1ms earlier would read as "predates the
          // newest publish".
          ranAt: new Date(Date.now()),
          ...overrides,
        })
        .run();
    }

    it('is ok when no installed module declares on_upstream_publish', () => {
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('not in use');
    });

    it('warns when a self-update module is installed but no publish has ever arrived', () => {
      insertHookModule('celilo-mgmt');
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('warn');
      expect(f.summary).toContain('no build-bus publish has ever arrived');
      expect(f.remediation).toContain('subscribers install-daemon');
      expect(f.remediation).toContain('CELILO_BUS_SECRET');
    });

    it('warns when publishes arrive but no hook has ever run', () => {
      insertHookModule('celilo-mgmt');
      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('warn');
      expect(f.summary).toContain('no self-update hook has ever run');
    });

    it('warns with the stderr tail when the newest run failed', () => {
      insertHookModule('celilo-mgmt');
      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
      seedRun({ exitCode: 1, stderrTail: 'bun add -g failed; not promoting' });
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('warn');
      expect(f.summary).toContain('exited 1');
      expect(f.detail.join('\n')).toContain('bun add -g failed');
    });

    it('warns when the newest run timed out', () => {
      insertHookModule('celilo-mgmt');
      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
      seedRun({ exitCode: null, timedOut: true, durationMs: 300000 });
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('warn');
      expect(f.summary).toContain('timed out');
    });

    it('warns when the newest run predates the newest publish — that publish dispatched nothing', () => {
      insertHookModule('celilo-mgmt');
      bus.emitRaw('build-bus.publish', { eventId: 'evt-2' });
      seedRun({ ranAt: new Date(now - 60 * MINUTE) });
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('warn');
      expect(f.summary).toContain('dispatched nothing');
    });

    it('is ok when the newest run succeeded and is fresh', () => {
      insertHookModule('celilo-mgmt');
      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
      seedRun();
      const f = checkBuildBusPublishing(bus, db, { now });
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('@celilo/cli@2.2.1');
    });
  });

  describe('checkSubscribers', () => {
    it('fails when a deployed module declares a subscription the bus is missing', () => {
      insertModule(
        'lunacycle',
        baseManifest({
          id: 'lunacycle',
          subscriptions: [{ name: 'smoke', pattern: 'deploy.completed.$self', handler: 'echo' }],
        }),
      );
      const f = checkSubscribers(bus, db);
      expect(f.status).toBe('fail');
      expect(f.autoFixable).toBe(true);
      expect(f.detail.join(' ')).toContain('lunacycle.smoke');
      expect(f.remediation).toContain('resync-subscriptions');
    });

    it('passes once the subscription is registered on the bus', () => {
      insertModule(
        'lunacycle',
        baseManifest({
          id: 'lunacycle',
          subscriptions: [{ name: 'smoke', pattern: 'deploy.completed.$self', handler: 'echo' }],
        }),
      );
      bus.subscribe({
        name: 'lunacycle.smoke',
        pattern: 'deploy.completed.lunacycle',
        handler: 'echo',
      });
      const f = checkSubscribers(bus, db);
      expect(f.status).toBe('ok');
    });

    it('warns about a stale subscriber with no deployed module', () => {
      // No modules deployed, but a leftover subscriber lingers.
      bus.subscribe({ name: 'ghost.sub', pattern: 'x', handler: 'echo', registeredBy: 'ghost' });
      const f = checkSubscribers(bus, db);
      expect(f.status).toBe('warn');
      expect(f.detail.join(' ')).toContain('ghost.sub');
      // `resync-subscriptions` never deletes, so it cannot clear this (#624).
      expect(f.autoFixable).toBe(false);
      expect(f.remediation).toContain('subscribers remove');
    });

    it('ignores core-registered subscribers that no manifest declares (#624)', () => {
      ensureSweepSubscriber(bus);
      ensureInboundSubscriber(bus);
      ensureBackupSweepSubscriber(bus);
      ensureOperationsSweepSubscriber(bus);
      const f = checkSubscribers(bus, db);
      expect(f.status).toBe('ok');
      expect(f.detail.join(' ')).not.toContain('celilo-');
    });

    // The guard for the invariant `checkSubscribers` classifies by, asserted
    // at the sites that ESTABLISH it rather than the one that consumes it —
    // so it keeps holding if the predicate is ever rewritten. Goes red the
    // moment a core registrar names its row under its own registrar id
    // (`celilo-alerting` registering `celilo-alerting.digest`), which would
    // read as a module row, find no module of that name, and be reported
    // stale — #624 again, and baffling to whoever hit it.
    //
    // ponytail: enumerates the registrars by hand because celilo has no
    // registry of them. A fifth one is not covered until it is added here;
    // if that ever bites, the upgrade is a shared `registerCoreSubscriber`
    // helper that every core site goes through, tested once.
    it('no core registrar produces a module-shaped subscriber name', () => {
      ensureSweepSubscriber(bus);
      ensureInboundSubscriber(bus);
      ensureBackupSweepSubscriber(bus);
      ensureOperationsSweepSubscriber(bus);

      const rows = bus.db
        .query<{ name: string; registered_by: string | null }, []>(
          'SELECT name, registered_by FROM subscribers',
        )
        .all();
      expect(rows.length).toBeGreaterThan(0);
      for (const row of rows) {
        expect(row.registered_by).not.toBeNull();
        expect(row.name.startsWith(`${row.registered_by}.`)).toBe(false);
      }
    });
  });

  describe('checkCapabilityProviders', () => {
    const consumer = (deriveFrom: string) =>
      baseManifest({
        id: 'forgejo',
        name: 'Forgejo',
        variables: {
          owns: [
            {
              name: 'idp_auth_url',
              type: 'string',
              required: true,
              source: 'capability',
              derive_from: deriveFrom,
            },
          ],
          imports: [],
        },
      });

    it('fails when the consumed capability has no deployed provider', () => {
      insertModule('forgejo', consumer('$capability:idp.auth_url'));
      const f = checkCapabilityProviders(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain("no deployed module provides 'idp'");
    });

    it('fails when the provider lacks the referenced field', () => {
      insertModule('forgejo', consumer('$capability:idp.auth_url'));
      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
      db.insert(capabilitiesTable)
        .values({
          moduleId: 'authentik',
          capabilityName: 'idp',
          version: '1.0.0',
          data: { admin_email: 'x' },
        })
        .run();
      const f = checkCapabilityProviders(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain('has no value there');
    });

    it('passes when the provider carries a concrete value', () => {
      insertModule('forgejo', consumer('$capability:idp.auth_url'));
      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
      db.insert(capabilitiesTable)
        .values({
          moduleId: 'authentik',
          capabilityName: 'idp',
          version: '1.0.0',
          data: { auth_url: 'https://auth.celilo.computer' },
        })
        .run();
      const f = checkCapabilityProviders(db);
      expect(f.status).toBe('ok');
    });

    it('passes but flags a derived ref for the chain trace (ISS-0114)', () => {
      // idp.auth_url is present but is itself a ref ($self:auth_url) — we can't
      // verify it resolves without the walker.
      insertModule('forgejo', consumer('$capability:idp.auth_url'));
      insertModule('authentik', baseManifest({ id: 'authentik', name: 'Authentik' }));
      db.insert(capabilitiesTable)
        .values({
          moduleId: 'authentik',
          capabilityName: 'idp',
          version: '1.0.0',
          data: { auth_url: '$self:auth_url' },
        })
        .run();
      const f = checkCapabilityProviders(db);
      expect(f.status).toBe('ok');
      // celilo#1308: `celilo capability chain` no longer exists, so the detail
      // says the verification is by hand rather than naming a command that
      // cannot run.
      expect(f.detail.join(' ')).toContain('verify these by hand');
      expect(f.detail.join(' ')).not.toContain('celilo capability chain');
    });
  });

  describe('checkServiceDns', () => {
    const NAT_IP = '192.168.0.253';

    function seedFirewall(moduleId: string, natIp: string): void {
      insertModule(moduleId, baseManifest({ id: moduleId, name: moduleId }));
      db.insert(capabilitiesTable)
        .values({ moduleId, capabilityName: 'firewall', version: '1.0.0', data: {} })
        .run();
      db.insert(moduleConfigs)
        .values({ moduleId, key: 'nat_ip', value: natIp, valueJson: JSON.stringify(natIp) })
        .run();
    }

    function seedSystem(moduleId: string, ip: string, zone: 'dmz' | 'internal'): void {
      db.insert(moduleSystems)
        .values({
          moduleId,
          name: 'main',
          hostname: moduleId,
          ipv4Address: ip,
          zone,
          infraType: 'container_service',
        })
        .run();
    }

    function seedRecord(provider: string, consumer: string, host: string, ip: string): void {
      db.insert(dnsInternalRecords)
        .values({ providerModuleId: provider, consumerModuleId: consumer, host, ip })
        .run();
    }

    it('is ok when no internal DNS records are registered', async () => {
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('no internal DNS records');
    });

    it('skips (ok) when records exist but no firewall natIp is configured', async () => {
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
      seedRecord('technitium', 'caddy', 'git.celilo.computer', '10.0.10.10');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
      expect(f.detail.join(' ')).toContain('no firewall');
    });

    it('is ok when a service record points at the natIp', async () => {
      seedFirewall('iptables', NAT_IP);
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
      seedRecord('technitium', 'caddy', 'git.celilo.computer', NAT_IP);
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
    });

    it('fails when a record points at a segmented-zone container IP', async () => {
      seedFirewall('iptables', NAT_IP);
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('forgejo', baseManifest({ id: 'forgejo', name: 'Forgejo' }));
      seedSystem('forgejo', '10.0.20.14', 'dmz');
      seedRecord('technitium', 'forgejo', 'git-ssh.git.celilo.computer', '10.0.20.14');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain('dmz-zone container IP');
      expect(f.remediation).toContain('natIp');
    });

    it('SKIPS `.infra.<zone>` system-identity records (intentionally container-IP)', async () => {
      seedFirewall('iptables', NAT_IP);
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('forgejo', baseManifest({ id: 'forgejo', name: 'Forgejo' }));
      seedSystem('forgejo', '10.0.20.14', 'dmz');
      // The on-system-event handler registers <host>.infra.<domain> at the
      // system's own container IP on purpose — a zone-side identity name, not a
      // LAN-reachability record. The check must NOT flag it.
      seedRecord('technitium', 'forgejo', 'forgejo.infra.celilo.computer', '10.0.20.14');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
    });

    it('is ok when a record points at an internal-zone (LAN) system IP', async () => {
      seedFirewall('iptables', NAT_IP);
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('homebridge', baseManifest({ id: 'homebridge', name: 'Homebridge' }));
      seedSystem('homebridge', '192.168.0.50', 'internal');
      seedRecord('technitium', 'homebridge', 'hb.celilo.computer', '192.168.0.50');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
    });

    it('warns when a record points at neither the natIp nor a known system IP', async () => {
      seedFirewall('iptables', NAT_IP);
      insertModule('technitium', baseManifest({ id: 'technitium', name: 'Technitium' }));
      insertModule('caddy', baseManifest({ id: 'caddy', name: 'Caddy' }));
      seedRecord('technitium', 'caddy', 'stale.celilo.computer', '203.0.113.9');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('warn');
      expect(f.detail.join(' ')).toContain('known system IP');
    });

    it('does not crash when the ledger table is missing (defers to schema check)', async () => {
      db.$client.run('DROP TABLE dns_internal_records');
      const f = await checkServiceDns(db);
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('ledger not present');
    });
  });

  describe('checkSchemaDrift', () => {
    it('is ok when every schema table is present (fresh migrated DB)', () => {
      const f = checkSchemaDrift(db);
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('schema tables');
    });

    it('fails and names a table the running CLI expects but the DB lacks', () => {
      db.$client.run('DROP TABLE dns_internal_records');
      const f = checkSchemaDrift(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain('dns_internal_records');
      expect(f.remediation).toContain('migrations');
    });

    // celilo#604: this is the state the rollout could not check. Every table
    // is present, one MIGRATED COLUMN is not, and the doctor must not call
    // that "migrations applied".
    it('fails and names a migrated COLUMN the DB lacks, with every table present', () => {
      db.$client.run('ALTER TABLE backups DROP COLUMN pid');
      const f = checkSchemaDrift(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain('backups.pid');
      expect(f.summary).not.toContain('present');
    });

    it('says it checked columns, not only tables', () => {
      const f = checkSchemaDrift(db);
      expect(f.status).toBe('ok');
      expect(f.summary).toContain('columns present');
    });

    it('fails when a journal migration has not been applied on this box', () => {
      db.$client.run(
        'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
      );
      const f = checkSchemaDrift(db);
      expect(f.status).toBe('fail');
      expect(f.detail.join(' ')).toContain('unapplied migration');
    });
  });
});

describe('findBrokenCapabilityDerivations (shared predicate)', () => {
  const consumer = baseManifest({
    id: 'forgejo',
    variables: {
      owns: [
        {
          name: 'idp_auth_url',
          type: 'string',
          required: true,
          source: 'capability',
          derive_from: '$capability:idp.auth_url',
        },
      ],
      imports: [],
    },
  });

  it('flags no-provider when the capability is absent from the map', () => {
    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {});
    expect(problems).toHaveLength(1);
    expect(problems[0].reason).toBe('no-provider');
    expect(describeCapabilityProblem(problems[0])).toContain("no deployed module provides 'idp'");
  });

  it('flags empty-value when the field is present but empty', () => {
    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
      idp: { auth_url: '' },
    });
    expect(problems[0].reason).toBe('empty-value');
  });

  it('flags unresolved-ref when the resolved value is still a template', () => {
    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
      idp: { auth_url: '$self:auth_url' },
    });
    expect(problems[0].reason).toBe('unresolved-ref');
    expect(problems[0].value).toBe('$self:auth_url');
  });

  it('returns nothing when the field resolves to a concrete value', () => {
    const problems = findBrokenCapabilityDerivations('forgejo', consumer, {
      idp: { auth_url: 'https://auth.celilo.computer' },
    });
    expect(problems).toHaveLength(0);
  });

  it('accepts a broken optional derivation when the manifest declares a fallback', () => {
    const withFallback = baseManifest({
      id: 'secondary-dns',
      variables: {
        owns: [
          {
            name: 'managed_domains',
            type: 'array',
            required: false,
            default: [],
            source: 'capability',
            derive_from: '$capability:dns_internal.dns.managed_domains',
          },
        ],
        imports: [],
      },
    });

    expect(findBrokenCapabilityDerivations('secondary-dns', withFallback, {})).toEqual([]);
    expect(
      findBrokenCapabilityDerivations('secondary-dns', withFallback, {
        dns_internal: { dns: { managed_domains: '$self:managed_domains' } },
      }),
    ).toEqual([]);
  });

  it('walks dotted paths', () => {
    const m = baseManifest({
      id: 'consumer',
      variables: {
        owns: [
          {
            name: 'primary',
            type: 'string',
            required: true,
            source: 'capability',
            derive_from: '$capability:dns_external.server.ip.primary',
          },
        ],
        imports: [],
      },
    });
    const ok = findBrokenCapabilityDerivations('consumer', m, {
      dns_external: { server: { ip: { primary: '1.2.3.4' } } },
    });
    expect(ok).toHaveLength(0);
    const broken = findBrokenCapabilityDerivations('consumer', m, {
      dns_external: { server: { ip: {} } },
    });
    expect(broken[0].reason).toBe('empty-value');
  });
});

describe('checkControlPlaneNetwork', () => {
  let db: DbClient;
  let cpDir: string;

  function setSubnet(zone: string, cidr: string) {
    db.insert(systemConfig)
      .values({ key: `network.${zone}.subnet`, value: cidr })
      .run();
  }

  function deployCeliloMgmt(zone: string, ip: string) {
    db.insert(modules)
      .values({
        id: 'celilo-mgmt',
        name: 'celilo-mgmt',
        version: '1.0.0',
        manifestData: {},
        sourcePath: '/tmp/celilo-mgmt',
      })
      .run();
    db.insert(moduleSystems)
      .values({
        moduleId: 'celilo-mgmt',
        name: 'main',
        hostname: 'celilo-mgr',
        ipv4Address: ip,
        zone: zone as 'internal' | 'secure-mgmt',
        infraType: 'machine',
      })
      .run();
  }

  beforeEach(async () => {
    cpDir = mkdtempSync(join(tmpdir(), 'fleet-cpn-'));
    db = await setupTestDatabaseAt(join(cpDir, 'celilo.db'));
  });

  afterEach(() => {
    try {
      rmSync(cpDir, { recursive: true, force: true });
    } catch {
      /* ignore */
    }
  });

  it('is ok when the control-plane network is a configured zone', () => {
    setSubnet('secure-mgmt', '10.0.120.0/24');
    deployCeliloMgmt('secure-mgmt', '10.0.120.10');

    const finding = checkControlPlaneNetwork(db);
    expect(finding.status).toBe('ok');
    expect(finding.summary).toContain('10.0.120.0/24');
    expect(finding.remediation).toBeNull();
  });

  it('warns — naming BOTH consequences — when the network is unrecognized', () => {
    // The production case: celilo-mgr on a network with no configured subnet.
    deployCeliloMgmt('secure-mgmt', '10.0.120.10');

    const finding = checkControlPlaneNetwork(db);
    expect(finding.status).toBe('warn');
    const detail = finding.detail.join(' ');
    // Silence is the bug; the report must name what actually breaks.
    expect(detail).toContain('firewall');
    expect(detail).toContain('resolver');
    expect(detail).toContain('hairpin');
    // ...and where it is, so the operator can act.
    expect(detail).toContain('10.0.120.10');
  });

  it('remediation names the concrete config key to set', () => {
    deployCeliloMgmt('secure-mgmt', '10.0.120.10');

    const finding = checkControlPlaneNetwork(db);
    expect(finding.remediation).toContain('network.secure-mgmt.subnet');
  });

  it('warns when celilo-mgmt is not deployed at all', () => {
    const finding = checkControlPlaneNetwork(db);
    expect(finding.status).toBe('warn');
    expect(finding.detail.join(' ')).toContain('no deployed systems');
  });

  it('is ok for the common case: control plane on the internal LAN', () => {
    setSubnet('internal', '192.168.0.0/24');
    deployCeliloMgmt('internal', '192.168.0.10');

    const finding = checkControlPlaneNetwork(db);
    expect(finding.status).toBe('ok');
    expect(finding.summary).toContain('192.168.0.0/24');
  });
});

/**
 * celilo#728. `system doctor` reported "OK with warnings" while a Proxmox node
 * was OFFLINE with `celilo-apt-repo` and `lunacycle` deployed on it. Its two
 * warnings were about the dispatcher and a false-positive subscriber drift —
 * neither related. The node's status was already in `proxmox node list`; doctor
 * never consulted it, and the condition surfaced only because a release run got
 * an HTTP 502 from the apt repo that happened to live there.
 */
describe('checkHostLiveness', () => {
  const inputs = (over: Partial<HostLivenessInputs> = {}): HostLivenessInputs => ({
    placements: [],
    machines: [],
    nodes: [],
    guestNodes: [],
    ...over,
  });

  it('reproduces #728: an offline node hosting modules is a FAILURE naming both', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [
          {
            moduleId: 'celilo-apt-repo',
            hostname: 'apt',
            infraType: 'container_service',
            vmid: 205,
          },
          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
        ],
        nodes: [
          { node: 'node2', online: false },
          { node: 'node3', online: true },
        ],
        guestNodes: [
          { vmid: 205, node: 'node2' },
          { vmid: 202, node: 'node2' },
          { vmid: 301, node: 'node3' },
        ],
      }),
    );

    expect(finding.status).toBe('fail');
    expect(finding.summary).toContain('node2');
    expect(finding.summary).toContain('2 module(s)');
    // Both the host AND what it takes down with it — the thing doctor could not say.
    expect(finding.detail.join('\n')).toContain('DOWN node2: celilo-apt-repo, lunacycle');
    // The healthy node is not implicated.
    expect(finding.summary).not.toContain('node3');
    expect(finding.remediation).toBeTruthy();
  });

  it('covers the machine pool, not only container-service nodes', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
        machines: [{ hostname: 'iot', reachable: false }],
      }),
    );
    expect(finding.status).toBe('fail');
    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
  });

  it('stays quiet when every host is up — no new permanent warning', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [
          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
          { moduleId: 'caddy', hostname: 'caddy', infraType: 'container_service', vmid: 301 },
        ],
        machines: [{ hostname: 'iot', reachable: true }],
        nodes: [{ node: 'node3', online: true }],
        guestNodes: [{ vmid: 301, node: 'node3' }],
      }),
    );
    expect(finding.status).toBe('ok');
    expect(finding.detail).toEqual([]);
    expect(finding.summary).toBe('2 host(s) up');
  });

  /**
   * The absent-vs-empty rule, one level down: "the cluster did not answer" must
   * never read as "every node is healthy".
   *
   * It WARNS rather than passing quietly. A cluster that will not answer its
   * own API is not evidence of health, and the failure to answer may be the
   * outage this check exists to catch — so reporting it as ok-with-a-note would
   * rebuild #728 one level down. The warning is safe to have precisely because
   * it is not permanent: a machine-only fleet produces no unverified hosts at
   * all, every placement resolving through the probe.
   */
  it('a host celilo tried and failed to verify WARNS, with the reason', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [
          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
        ],
        // The cluster was unreachable, so nothing came back about vmid 202.
      }),
    );
    expect(finding.status).toBe('warn');
    expect(finding.detail.join('\n')).toContain('lunacycle');
    expect(finding.detail.join('\n')).toContain("not present in the cluster's resources");
    expect(finding.remediation).toBeTruthy();
  });

  it('an unprobed machine warns, never assumed reachable', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
        machines: [{ hostname: 'somethingelse', reachable: true }],
      }),
    );
    expect(finding.status).toBe('warn');
    expect(finding.detail.join('\n')).toContain('no probe result for this machine');
  });

  /**
   * The reason is per host, not one blanket sentence: "could not verify" tells
   * an operator nothing about whether to go and look at a cluster, a machine,
   * or a stale row.
   */
  it('names a different reason for each way verification can fail', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [
          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
          { moduleId: 'droplet-app', hostname: 'vps', infraType: 'container_service', vmid: null },
        ],
      }),
    );
    expect(finding.status).toBe('warn');
    const detail = finding.detail.join('\n');
    expect(detail).toContain('no probe result for this machine');
    // The shape a non-Proxmox provider takes today, until celilo can read it.
    expect(detail).toContain('no liveness source for this provider');
  });

  it('a down host still fails when a sibling host is unverified', () => {
    const finding = checkHostLiveness(
      inputs({
        placements: [
          { moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null },
          { moduleId: 'lunacycle', hostname: 'luna', infraType: 'container_service', vmid: 202 },
        ],
        machines: [{ hostname: 'iot', reachable: false }],
      }),
    );
    expect(finding.status).toBe('fail');
    expect(finding.detail.join('\n')).toContain('DOWN iot: homebridge');
    expect(finding.detail.join('\n')).toContain('unverified');
  });

  it('is silent on a fleet with nothing deployed', () => {
    const finding = checkHostLiveness(inputs());
    expect(finding.status).toBe('ok');
    expect(finding.detail).toEqual([]);
  });
});

/**
 * The wiring, which is what actually fixes celilo#728. A pure check nothing
 * calls changes nothing: on `main` today `runFleetChecks` returns no
 * host-liveness finding at all, which is precisely why doctor said
 * "OK with warnings" over an offline node.
 */
describe('runFleetChecks includes host liveness (#728)', () => {
  let dir: string;
  let db: DbClient;
  let bus: Bus;

  beforeEach(async () => {
    dir = mkdtempSync(join(tmpdir(), 'fleet-liveness-'));
    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
    process.env.EVENT_BUS_DB = join(dir, 'events.db');
    db = await setupTestDatabaseAt(join(dir, 'celilo.db'));
    bus = openBus({ dbPath: join(dir, 'events.db'), events: defineEvents({}) });
  });

  afterEach(() => {
    bus.close();
    db.$client.close();
    resetTestDbPath();
    delete process.env.EVENT_BUS_DB;
    try {
      rmSync(dir, { recursive: true, force: true });
    } catch {
      /* ignore */
    }
  });

  it('surfaces an offline node through the doctor findings, not just the checker', async () => {
    const findings = await runFleetChecks(bus, db, {
      hostLiveness: async () => ({
        placements: [
          {
            moduleId: 'celilo-apt-repo',
            hostname: 'apt',
            infraType: 'container_service',
            vmid: 205,
          },
        ],
        machines: [],
        nodes: [{ node: 'node2', online: false }],
        guestNodes: [{ vmid: 205, node: 'node2' }],
      }),
    });

    const liveness = findings.find((f) => f.id === 'host-liveness');
    // Against main this is `undefined` — the check does not exist in the list.
    expect(liveness).toBeDefined();
    expect(liveness?.status).toBe('fail');
    expect(liveness?.summary).toContain('node2');
  });

  it('does not add a standing warning to a healthy fleet', async () => {
    const findings = await runFleetChecks(bus, db, {
      hostLiveness: async () => ({
        placements: [{ moduleId: 'homebridge', hostname: 'iot', infraType: 'machine', vmid: null }],
        machines: [{ hostname: 'iot', reachable: true }],
        nodes: [],
        guestNodes: [],
      }),
    });
    const liveness = findings.find((f) => f.id === 'host-liveness');
    expect(liveness?.status).toBe('ok');
  });
});