#!/usr/bin/env bun
/**
 * E2E Test Runner
 *
 * Runs each test file sequentially with live progress display.
 * Tracks historical timing in .e2e-timing.json for countdown ETAs.
 * Detects false passes (0 assertions).
 *
 * Progress protocol:
 *   [progress:start] doing message | done message
 *   [progress:done] optional override
 *   [progress:fail] error message
 *
 * Display:
 *   While in progress:  "  -45s  … deploying caddy"
 *   When done:          "  -45s  ✔ caddy deployed"
 *   Countdown goes negative when over estimate.
 */

import { execSync, spawn, spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
import { basename, dirname, join, resolve } from 'node:path';
import { type DisplayMode, ProgressDisplay } from '@celilo/cli-display';
import { parse as parseYaml } from 'yaml';
import {
  type BlockTiming,
  type OverBudgetBlock,
  declaredBlockCaps,
  mergeBlockTiming,
  overBudgetBlocks,
  parseBlockDurations,
  serializeBlockTiming,
  tightBlocks,
} from './block-timing';
import {
  emitRunCompleted,
  emitRunFailed,
  emitRunStarted,
  emitTestCompleted,
  emitTestStarted,
} from './bus-events';
import { diagnose, formatReport } from './doctor';
import { type StageTally, extractFailureMessage, stripAnsi, tallyStages } from './extract-failure';
import { writeLastRun } from './last-run';
import { parseLine } from './parse-line';
import { KNOWN_RUN_FLAGS, filterTestFilesByPatterns, parseRunArgs } from './run-args';
import { E2eBusyError, acquireRunLock, markKept } from './run-lock';
import { SIMULATOR_IPS } from './simulator-ips';

// ─── Config ──────────────────────────────────────────────────────────

const PKG_DIR = resolve(import.meta.dir, '..');
const testDir = process.env.E2E_TEST_DIR || process.cwd();
/** Set by `cele2e run <module-path>` to run tests from that module's e2e/ dir */
const moduleDir = process.env.E2E_MODULE_DIR;
/** Set by `cele2e run` (no args) when invoked outside a module — runs every discovered module's tests. */
const moduleDirs = process.env.E2E_MODULE_DIRS
  ? process.env.E2E_MODULE_DIRS.split('\n').filter(Boolean)
  : moduleDir
    ? [moduleDir]
    : [];

function resolveModuleTestsDir(dir: string): string {
  const manifestPath = join(dir, 'manifest.yml');
  if (existsSync(manifestPath)) {
    try {
      const parsed = parseYaml(readFileSync(manifestPath, 'utf-8')) as {
        e2e?: { tests_dir?: string };
      };
      if (parsed?.e2e?.tests_dir) return resolve(dir, parsed.e2e.tests_dir);
    } catch {}
  }
  return join(dir, 'e2e');
}

const testsPath = moduleDir
  ? resolveModuleTestsDir(moduleDir)
  : (process.env.E2E_TESTS_PATH ?? join(testDir, 'tests'));
/** Set by `cele2e run --all` so a module-dirs run ALSO picks up the repo's top-level e2e/tests/. */
const topLevelTestsPath = process.env.E2E_TOP_LEVEL_TESTS;
const timingFile = join(testDir, '.e2e-timing.json');
/** Per-test-block durations. The 300s cap is per BLOCK, and nothing else records that. */
const blockTimingFile = join(testDir, '.e2e-block-timing.json');
const persistentFile = join(testDir, '.e2e-persistent.json');

// Parse flags out of argv
const rawArgs = process.argv.slice(2);
const flagKeep = rawArgs.includes('--keep');
const flagReuse = rawArgs.includes('--reuse');
const flagLive = rawArgs.includes('--live');
const flagPublished = rawArgs.includes('--published');
// --source-cli: run the celilo CLI from the mounted workspace instead of the one
// baked into the management image. Off by default because the baked CLI is the
// artifact under test, and running the source over the bind mount roughly
// doubles every celilo command's start-up (measured 0.14s vs 0.31s on virtiofs,
// 0.16s vs 0.56s on sshfs), which a 25-40 command test pays each time. The
// image's shim reads this from the container environment; the compose generator
// forwards it. Use it to iterate without a build-infra.
const flagSourceCli = rawArgs.includes('--source-cli');
const flagVerbose = rawArgs.includes('--verbose');
// CI mode: no animated spinner, no [progress:*] markers — just one clean
// ✔/✗ line per step (plus sub-events) written straight to the log. Forgejo's
// runner allocates a PTY, so process.stdout.isTTY is truthy there and the
// interactive footer (redrawn via `\r` on every spinner tick) gets linearized
// into one log line per tick — dozens of identical braille lines per step.
// `--ci` (or any standard `CI` env var, e.g. Forgejo/GitHub Actions) forces
// the non-interactive render path that sidesteps that entirely. `--no-ci`
// opts back out when a CI env var is set but an operator wants the spinner.
const flagCi = rawArgs.includes('--ci');
const flagNoCi = rawArgs.includes('--no-ci');
const ci = flagCi || (!flagNoCi && !!process.env.CI);
const RUN_USAGE = `
Usage: cele2e run [pattern...] [flags]

Runs e2e suites. Each pattern is a substring matched against test file
names; no pattern (bare) runs every suite in the current scope.

Flags:
  --keep            Keep network running after tests
  --reuse           Reuse existing network if running
  --live            Use live (non-simulated) internet
  --published       Use published .netapp packages
  --source-cli      Run the celilo CLI from the mounted workspace
  --verbose         Verbose output
  --ci / --no-ci    Force plain CI log output on/off (auto-on when CI env var set)
  --complete        Include quarantined (cele2e-ci-unsafe) tests
  --notify          Desktop notification when the run finishes
  --shuffle [--seed=<n>]  Randomize test order; replay an order with its seed

Full CLI help: cele2e --help   (run --all is a cele2e-level alias, not a runner flag)
`;

const parsedArgs = parseRunArgs(rawArgs);
if (parsedArgs.helpRequested) {
  console.log(RUN_USAGE);
  process.exit(0);
}
if (parsedArgs.unknownFlags.length > 0) {
  console.error(`Unknown flag(s): ${parsedArgs.unknownFlags.join(' ')}`);
  console.error(
    `Known flags: ${[...KNOWN_RUN_FLAGS, '--seed=<n>'].join(' ')}   (--help for run usage)`,
  );
  process.exit(2);
}
const patterns = parsedArgs.patterns;
// Quarantine: tests whose file contains the marker `cele2e-ci-unsafe` (a comment
// naming the reason + ISS) are known-failing / builder-flaky and tracked
// separately, so they must never turn a broad run red. They are SKIPPED by
// default in every broad run (`--all`, `--all-modules`, `--ci-safe`, bare). Two
// ways to run one anyway: name it explicitly (`cele2e run <name>`), or pass
// `--complete` (the deliberate "run everything, including quarantined" opt-in).
// `--complete` is passed through from the CLI (not stripped) so the runner sees it.
const flagComplete = rawArgs.includes('--complete');
// --notify: fire a desktop notification when the run finishes (handy for long
// detached/background runs). Best-effort — never affects the exit code.
const flagNotify = rawArgs.includes('--notify');
// --shuffle [--seed=N]: randomize test order to surface order-dependent bugs
// (e2e-confidence #258). The install-sh DNS-pollution bug only manifested in a
// specific order — a shuffle lane flushes that class out. Seeded + logged so a
// failing order is reproducible: replay with the printed --seed=N.
const flagShuffle = rawArgs.includes('--shuffle');
const seedFlag = rawArgs.find((a) => a.startsWith('--seed='));
const seedArg = seedFlag ? Number.parseInt(seedFlag.split('=')[1] ?? '', 10) : undefined;

/** Deterministic Fisher-Yates shuffle seeded by a 32-bit value (mulberry32), so
 * a randomized test order is reproducible via --seed=<n>. */
function seededShuffle<T>(arr: readonly T[], seed: number): T[] {
  let s = seed >>> 0;
  const rand = (): number => {
    s = (s + 0x6d2b79f5) >>> 0;
    let t = s;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
  const out = [...arr];
  for (let i = out.length - 1; i > 0; i--) {
    const j = Math.floor(rand() * (i + 1));
    [out[i], out[j]] = [out[j], out[i]];
  }
  return out;
}

function notify(title: string, message: string): void {
  if (!flagNotify) return;
  try {
    if (process.platform === 'darwin') {
      const r = spawnSync(
        'terminal-notifier',
        ['-title', title, '-message', message, '-sound', 'Glass'],
        { stdio: 'ignore' },
      );
      if (r.error) {
        spawnSync(
          'osascript',
          [
            '-e',
            `display notification ${JSON.stringify(message)} with title ${JSON.stringify(title)} sound name "Glass"`,
          ],
          { stdio: 'ignore' },
        );
      }
      spawnSync('say', [message], { stdio: 'ignore' });
    } else if (process.platform === 'linux') {
      spawnSync('notify-send', [title, message], { stdio: 'ignore' });
    }
  } catch {
    // ponytail: notification is a nicety; its failure must never break a run.
  }
}

// ─── Colors ──────────────────────────────────────────────────────────

const bold = '\x1b[1m';
const dim = '\x1b[2m';
const green = '\x1b[32m';
const red = '\x1b[31m';
const yellow = '\x1b[33m';
const reset = '\x1b[0m';

const interactive =
  process.stdout.isTTY && !process.argv.includes('--no-interactive') && !flagVerbose && !ci;

// ─── Timing ──────────────────────────────────────────────────────────

type TimingHistory = Record<string, number>;

function loadTiming(): TimingHistory {
  try {
    return JSON.parse(readFileSync(timingFile, 'utf-8'));
  } catch {
    return {};
  }
}

function saveTiming(history: TimingHistory): void {
  writeFileSync(timingFile, `${JSON.stringify(history, Object.keys(history).sort(), 2)}\n`);
}

function loadBlockTiming(): BlockTiming {
  try {
    return JSON.parse(readFileSync(blockTimingFile, 'utf-8'));
  } catch {
    return {};
  }
}

function saveBlockTiming(timing: BlockTiming): void {
  writeFileSync(blockTimingFile, serializeBlockTiming(timing));
}

function formatDuration(secs: number): string {
  if (secs >= 60) return `${Math.floor(secs / 60)}m${Math.abs(secs) % 60}s`;
  return `${secs}s`;
}

// ─── Error Extraction ────────────────────────────────────────────────
// extractFailureMessage / stripAnsi live in extract-failure.ts: this file
// calls main() at import time, so nothing declared here is testable.

// ─── Test Execution ──────────────────────────────────────────────────

interface TestResult {
  name: string;
  status: 'pass' | 'fail' | 'suspicious';
  duration: number;
  error?: string;
  /** Last ~30 lines of raw test output, shown on failure so the user doesn't have to dig in logs */
  rawTail?: string;
  hadDebugPause?: boolean;
  projectName?: string;
  /** Per-`test()` durations in ms, so the per-block cap has a durable record. */
  blocks?: Record<string, number>;
  /**
   * Real stage failures vs stages a failing earlier stage blocked. Without the
   * split, one bad fixture line in stage 1 of a 10-stage suite reports as "9
   * failed" — nine counts of a defect that does not exist.
   */
  stages?: StageTally;
  /**
   * The test child exited 3: the live-stack guard refused to clean up because
   * another run's stack is in the way (celilo#1297). Bun itself never exits 3
   * — it uses 1 for failures — so the code is unambiguous. The run stops here:
   * every remaining suite would hit the same refusal, and continuing would
   * only pile more noise onto one cause.
   */
  refused?: boolean;
}

async function runTest(
  file: string,
  name: string,
  expectedDuration: number | undefined,
  logDir: string,
  display: ProgressDisplay,
): Promise<TestResult> {
  const start = Date.now();
  const logFile = join(logDir, 'output.log');
  const junitFile = join(logDir, 'junit.xml');

  display.reset(start, expectedDuration);
  let hadDebugPause = false;
  let capturedProjectName: string | undefined;

  return new Promise<TestResult>((resolveTest) => {
    // Env vars let the test detect --keep / --reuse / --live / --published modes
    const testEnv: NodeJS.ProcessEnv = { ...process.env };
    if (flagKeep) testEnv.CELILO_E2E_KEEP = '1';
    if (flagLive) {
      // Resolve infra root relative to the test directory (sibling repo convention)
      const inferredRoot = resolve(testDir, '../../infra');
      testEnv.CELILO_E2E_INFRA_ROOT = existsSync(join(inferredRoot, 'apps', 'celilo'))
        ? inferredRoot
        : resolve(PKG_DIR, '..', '..'); // fallback: @celilo/e2e is inside the infra monorepo
    }
    if (flagPublished) {
      testEnv.CELILO_E2E_INFRA_ROOT = ''; // empty = force published mode
    }
    if (flagSourceCli) testEnv.CELILO_E2E_SOURCE_CLI = '1';
    if (flagReuse) {
      // Read the persistent project and pass it to the test
      try {
        const data = JSON.parse(readFileSync(persistentFile, 'utf-8'));
        testEnv.CELILO_E2E_REUSE = '1';
        testEnv.CELILO_E2E_PROJECT = data.projectName;
      } catch {
        console.error(
          `${red}✗ --reuse: no persistent network found. Run with --keep first.${reset}`,
        );
        process.exit(1);
      }
    }

    // 1 hour timeout per test (debug sessions can be long). Each test() block's
    // own third argument overrides this — measured on bun 1.3.3, including when
    // the body swallows its own errors — so this is a backstop, not the budget.
    //
    // The JUnit report is the ONLY source of a duration for a block that PASSED:
    // bun's console names a block only when it fails. It does not change the
    // console output, so the display and the failure parsers are untouched.
    const proc = spawn(
      'bun',
      ['test', '--timeout', '3600000', '--reporter=junit', `--reporter-outfile=${junitFile}`, file],
      {
        cwd: PKG_DIR,
        stdio: ['pipe', 'pipe', 'pipe'],
        env: testEnv,
      },
    );

    let stdout = '';
    let stderr = '';
    const lines: string[] = [];

    function handleDebugPause(project: string, container: string) {
      display.instantEvent(`${bold}\x1b[33m⏸ DEBUG: paused at ${container}${reset}`);

      const signalFile = join(PKG_DIR, `.debug-resume-${proc.pid}`);
      const border = '═'.repeat(56);
      process.stdout.write(`\n${bold}\x1b[33m╔${border}${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m║  DEBUG: test paused${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m╠${border}${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m║${reset}  Project:   ${bold}${project}${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m║${reset}  Container: ${bold}${container}${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m║${reset}\n`);
      process.stdout.write(`${bold}\x1b[33m║${reset}  ${dim}Run commands:${reset}\n`);
      process.stdout.write(
        `${bold}\x1b[33m║${reset}    docker compose -f docker-compose.test.yml \\\n`,
      );
      process.stdout.write(
        `${bold}\x1b[33m║${reset}      -p ${project} exec ${container} bash -c '<cmd>'\n`,
      );
      process.stdout.write(`${bold}\x1b[33m║${reset}\n`);

      if (interactive) {
        process.stdout.write(
          `${bold}\x1b[33m║${reset}  ${dim}Or get a shell: type 'exit' to resume the test.${reset}\n`,
        );
        process.stdout.write(`${bold}\x1b[33m╚${border}${reset}\n\n`);

        spawnSync(
          'docker',
          ['compose', '-f', 'docker-compose.test.yml', '-p', project, 'exec', container, 'bash'],
          {
            cwd: PKG_DIR,
            stdio: 'inherit',
          },
        );
      } else {
        process.stdout.write(
          `${bold}\x1b[33m║${reset}  ${dim}Resume: touch ${signalFile}${reset}\n`,
        );
        process.stdout.write(`${bold}\x1b[33m╚${border}${reset}\n\n`);

        // Non-interactive: just wait for the signal file (no shell)
        // The caller creates the file when they're done investigating.
        return;
      }

      process.stdout.write(`\n${dim}Resuming test...${reset}\n\n`);
      writeFileSync(signalFile, 'resume');
    }

    function processChunk(chunk: Buffer) {
      const text = chunk.toString();
      stdout += text;

      for (const part of text.split('\n')) {
        const trimmed = part.trim();
        if (!trimmed) continue;
        lines.push(trimmed);

        // Check for debug pause signal: [debug:pause] <project> <container>
        const debugMatch = trimmed.match(/\[debug:pause\] (\S+) (\S+)/);
        if (debugMatch) {
          hadDebugPause = true;
          handleDebugPause(debugMatch[1], debugMatch[2]);
          continue;
        }

        // Capture project name for persistent network tracking
        const projectMatch = trimmed.match(/\[e2e:project\] (\S+)/);
        if (projectMatch) {
          capturedProjectName = projectMatch[1];
          continue;
        }

        // Detect bun test failure so pending progress steps show ✗ instead of ✔
        if (/^\s*\(fail\)\s/.test(trimmed) || /^\s*✗\s+.+\[/.test(trimmed)) {
          display.markTestFailing();
        }

        parseLine(trimmed, display, { verbose: flagVerbose });
      }
    }

    proc.stdout.on('data', processChunk);
    proc.stderr.on('data', (chunk: Buffer) => {
      stderr += chunk.toString();
      processChunk(chunk);
    });

    proc.on('close', (code) => {
      const duration = Math.floor((Date.now() - start) / 1000);
      if (code === 0) {
        display.flush();
      } else {
        display.abandonFailed();
      }

      writeFileSync(logFile, `${stdout}\n---STDERR---\n${stderr}`);

      const expectMatch = stdout.match(/(\d+) expect\(\) calls/);
      const expectCount = expectMatch ? Number.parseInt(expectMatch[1], 10) : undefined;
      const blocks = parseBlockDurations(lines, readJunit(junitFile));

      if (code === 0) {
        if (expectCount === 0) {
          resolveTest({
            name,
            status: 'suspicious',
            duration,
            blocks,
            hadDebugPause,
            projectName: capturedProjectName,
            error: '0 assertions — test likely exited before reaching assertions',
          });
        } else {
          resolveTest({
            name,
            status: 'pass',
            duration,
            blocks,
            hadDebugPause,
            projectName: capturedProjectName,
          });
        }
      } else {
        // Collect a raw tail: strip ANSI, drop internal protocol lines, keep last 30
        const rawTail = (stdout + stderr)
          .split('\n')
          .map(stripAnsi)
          .map((l) => l.trimEnd())
          .filter(
            (l) =>
              l.length > 0 &&
              !l.startsWith('[progress') &&
              !l.startsWith('[e2e:') &&
              !l.startsWith('[ansible:') &&
              !l.startsWith('[debug:'),
          )
          .slice(-30)
          .join('\n');
        resolveTest({
          name,
          status: 'fail',
          duration,
          blocks,
          hadDebugPause,
          projectName: capturedProjectName,
          error: extractFailureMessage(lines, code ?? 1),
          rawTail,
          stages: tallyStages(lines),
          refused: code === 3,
        });
      }
    });
  });
}

/** bun writes this only if it got far enough to; a crash leaves nothing. */
function readJunit(path: string): string | undefined {
  try {
    return readFileSync(path, 'utf-8');
  } catch {
    return undefined;
  }
}

/**
 * Print any block that finished with less than 20% of its own budget left.
 *
 * This runs HERE, in the runner, because it is the only place that sees real
 * durations under real load. A gate in CI cannot see this failure mode at all:
 * on a quiet runner every block fits, and the run is green (celilo#1268,
 * acceptance 4).
 *
 * It reports rather than fails. A block at 90% has not broken anything yet, and
 * failing a passing suite would stop the grind that produces the measurements.
 * The line is the durable signal — it lands in `output.log`, so `classify.sh`
 * can promote a PASS whose blocks are out of room, instead of a later timeout
 * arriving with nothing to distinguish "the host starved this" from "this never
 * had room".
 */
/**
 * Fail a suite whose block overran the cap that block itself declares.
 *
 * Reads the JUnit report directly — `overBudgetBlocks` bypasses the skip filter
 * for the reason stated there (celilo#1291: the filter deleted the one block
 * that mattered). bun's per-test timeout is not the enforcer; this is. A block
 * that overruns while the suite still PASSES is the dangerous case — an
 * unbudgeted lease on the rig that reads as green — so a pass with an overrun
 * becomes a fail naming the block, its duration and its declaration. A failed
 * suite keeps its own failure; the overruns are printed either way so the
 * output.log carries the durable line. Debug sessions are exempt: a
 * `net.debug()` pause inside a block inflates its junit time without the suite
 * having run at all.
 */
function enforceBlockCaps(result: TestResult, suite: string, logDir: string, file: string): void {
  if (result.hadDebugPause) return;
  const junitXml = readJunit(join(logDir, 'junit.xml'));
  if (!junitXml) return;
  let overruns: OverBudgetBlock[];
  try {
    overruns = overBudgetBlocks(junitXml, readFileSync(file, 'utf-8'));
  } catch {
    return;
  }
  if (overruns.length === 0) return;
  for (const t of overruns) {
    console.log(
      `${red}[block:over-budget]${reset} ${suite} — "${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s declaration (${Math.round(t.fraction * 100)}%)`,
    );
  }
  if (result.status === 'pass') {
    result.status = 'fail';
    result.error = overruns
      .map(
        (t) =>
          `"${t.block}" ran ${Math.round(t.ms / 1000)}s against its ${Math.round(t.capMs / 1000)}s cap`,
      )
      .join('; ');
  }
}

function reportTightBlocks(suite: string, file: string, blocks: Record<string, number>): void {
  let caps: Record<string, number>;
  try {
    caps = declaredBlockCaps(readFileSync(file, 'utf-8'));
  } catch {
    return;
  }
  for (const t of tightBlocks(blocks, caps)) {
    const pct = Math.round(t.fraction * 100);
    console.log(
      `${yellow}[block:tight]${reset} ${suite} — "${t.block}" used ${Math.round(t.ms / 1000)}s of its ${Math.round(t.capMs / 1000)}s budget (${pct}%)`,
    );
  }
}

/**
 * " — 1 stage failed, 8 skipped (blocked by an earlier stage)". Rendered only
 * when a cascade actually happened, so an ordinary single-stage failure stays
 * as terse as it was.
 */
function stageSuffix(stages: StageTally | undefined): string {
  if (!stages || stages.skipped === 0) return '';
  const failed = `${stages.failed} stage${stages.failed === 1 ? '' : 's'} failed`;
  return ` ${dim}— ${failed}, ${stages.skipped} skipped (blocked by an earlier stage)${reset}`;
}

// ─── Shared Infrastructure ───────────────────────────────────────────

function dockerComposeShared(cmd: string): string {
  try {
    return execSync(`docker compose -f docker-compose.shared.yml -p celilo-e2e-shared ${cmd}`, {
      cwd: PKG_DIR,
      encoding: 'utf-8',
      stdio: ['pipe', 'pipe', 'pipe'],
      timeout: 60_000,
    }).trim();
  } catch {
    return '';
  }
}

function stopSharedInfra(): void {
  dockerComposeShared('down --volumes --remove-orphans');
  try {
    execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 });
  } catch {}
}

/**
 * Detect cross-test pollution of the SHARED authoritative DNS at the test
 * boundary (e2e-confidence #254). A test that leaves a protected sim record
 * clobbered — canonically celilo.computer's apex off the website-sim, the bug
 * that silently broke install-sh ~15 tests downstream — has violated the shared
 * contract. Checked AFTER each test so the polluter is named HERE, at the cause,
 * instead of a later test failing mysteriously. On detection we re-scrub
 * (cp /seed -> /config, mirroring scrubDnsZones) so the damage doesn't cascade.
 *
 * Returns a description of the violation (and heals) if polluted, else null.
 * A query failure (infra down / tool missing) is not treated as pollution.
 */
function detectSharedDnsPollution(): string | null {
  const served = dockerComposeShared(
    `exec -T namecheap-dns sh -c ${JSON.stringify('kdig @127.0.0.1 celilo.computer A +short')}`,
  ).trim();
  if (!served) return null;
  if (served.split(/\s+/).includes(SIMULATOR_IPS.WEBSITE)) return null;
  // Polluted — heal so the next test starts from a clean baseline.
  dockerComposeShared(
    `exec -T namecheap-dns sh -c ${JSON.stringify('cp -f /seed/*.zone /config/ && knotc zone-reload')}`,
  );
  return `celilo.computer apex resolves to "${served}" (expected the website-sim ${SIMULATOR_IPS.WEBSITE})`;
}

// ─── Docker Check ────────────────────────────────────────────────────

function assertDockerAvailable(): void {
  try {
    execSync('docker info', { stdio: 'pipe', timeout: 5_000 });
  } catch {
    console.error(`\n${red}✗ Docker is not running.${reset}`);
    console.error(`  Start it with: ${bold}colima start${reset}`);
    console.error(`  Then retry:    ${bold}cele2e run${reset}\n`);
    process.exit(1);
  }
}

// ─── Main ────────────────────────────────────────────────────────────

async function main() {
  assertDockerAvailable();

  // Serialize against other cele2e runs sharing the global Docker infra. Fail
  // fast naming the holder; pollers use `cele2e status`. Acquire BEFORE any
  // docker mutation so a competing run can't wipe our shared infra mid-setup.
  const lockLabel = patterns.length ? patterns.join(',') : moduleDirs.length ? 'modules' : 'all';
  try {
    const outcome = acquireRunLock({
      test: lockLabel,
      runId: process.env.CELE2E_RUN_ID ?? ambientRunId,
      allowKept: flagReuse,
    });
    if (outcome.autoReleasedOwnKept) {
      const own = outcome.autoReleasedOwnKept;
      console.log(
        `${dim}Auto-released your own kept stack (${own.test}, held since ${own.startedAt}) and continuing.${reset}`,
      );
      console.log(
        `${dim}Use \`cele2e run --reuse\` instead if you meant to run against it.${reset}`,
      );
    }
    // A --reuse run keeps using a kept stack that stays up afterwards — hold the
    // lock in kept state so it isn't freed out from under the reused network.
    if (flagReuse) markKept();
  } catch (err) {
    if (err instanceof E2eBusyError) {
      console.error(`\n${red}${err.message}${reset}`);
      console.error(
        `${dim}Poll with: cele2e status   (machine-readable: cele2e status --json)${reset}\n`,
      );
      process.exit(3);
    }
    throw err;
  }

  // Preflight the environment BEFORE touching Docker. Every check here failed
  // silently once and surfaced later somewhere unrelated — a missing bake as an
  // SSH error against a firewall IP, a pruned base image as a TLS timeout 16
  // images into a build. Refusing now costs seconds; not refusing cost a full
  // run plus a debugging cycle each time. Lock state is excluded: we already
  // hold the lock, so checkRunLock would report ourselves as contention.
  const health = diagnose({ pkgDir: PKG_DIR, skipLock: true });
  if (!health.ok) {
    console.error(
      `\n${red}✗ cele2e preflight failed — not starting a run that cannot succeed.${reset}`,
    );
    for (const line of formatReport(health)) console.error(line);
    console.error(`\n${dim}Full environment report: cele2e doctor${reset}\n`);
    process.exit(1);
  }
  for (const c of health.checks) {
    if (c.status === 'warn') console.log(`${dim}! ${c.name}: ${c.detail}${reset}`);
  }

  let testFiles: string[] = [];

  if (moduleDirs.length > 0) {
    // One or more modules — collect tests from each module's e2e dir.
    for (const dir of moduleDirs) {
      if (!existsSync(join(dir, 'manifest.yml'))) {
        console.error(`${red}Not a celilo module: no manifest.yml found at ${dir}${reset}`);
        process.exit(1);
      }
      const tp = resolveModuleTestsDir(dir);
      if (!existsSync(tp)) {
        console.error(`${red}No tests directory found at ${tp}${reset}`);
        console.error(
          `${red}Set e2e.tests_dir in ${join(dir, 'manifest.yml')} to point to your test files.${reset}`,
        );
        process.exit(1);
      }
      // Auto-install e2e/ dependencies if package.json exists but node_modules is missing
      const e2ePkg = join(tp, 'package.json');
      const e2eNodeModules = join(tp, 'node_modules');
      if (existsSync(e2ePkg) && !existsSync(e2eNodeModules)) {
        process.stdout.write(`${dim}Installing e2e dependencies for ${basename(dir)}...${reset}\n`);
        execSync('bun install', { cwd: tp, stdio: 'inherit' });
        process.stdout.write('\n');
      }
      const files = readdirSync(tp)
        .filter((f) => f.endsWith('.test.ts'))
        .sort()
        .map((f) => resolve(tp, f));
      testFiles.push(...files);
    }
    // `cele2e run --all` sets E2E_MODULE_DIRS (every module) AND
    // E2E_TOP_LEVEL_TESTS (the repo's top-level e2e/tests/), so one invocation
    // covers BOTH the module suites and the top-level tests. Without this, the
    // module branch and the testsPath branch below are mutually exclusive and
    // no single run can cover everything (ISS-0013).
    if (topLevelTestsPath) {
      if (!existsSync(topLevelTestsPath)) {
        console.error(`${red}No top-level tests directory found at ${topLevelTestsPath}${reset}`);
        process.exit(1);
      }
      const files = readdirSync(topLevelTestsPath)
        .filter((f) => f.endsWith('.test.ts'))
        .sort()
        .map((f) => resolve(topLevelTestsPath, f));
      testFiles.push(...files);
    }
  } else {
    if (!existsSync(testsPath)) {
      console.error(`${red}No tests/ directory found at ${testsPath}${reset}`);
      process.exit(1);
    }
    testFiles = readdirSync(testsPath)
      .filter((f) => f.endsWith('.test.ts'))
      .sort()
      .map((f) => resolve(testsPath, f));
    // Docker-backed integration tests live in a sibling `tests-integration/`
    // so they stay OUT of the hermetic `bun test tests/` gate. cele2e is the
    // heavy runner (Docker + build-infra present), so it covers BOTH tiers —
    // this is what keeps the documented `cele2e run ansible-output` working
    // after the test moved out of tests/.
    const integrationPath = join(dirname(testsPath), 'tests-integration');
    if (existsSync(integrationPath)) {
      testFiles.push(
        ...readdirSync(integrationPath)
          .filter((f) => f.endsWith('.test.ts'))
          .sort()
          .map((f) => resolve(integrationPath, f)),
      );
    }
  }

  if (patterns.length > 0) {
    testFiles = filterTestFilesByPatterns(testFiles, patterns);
  }

  // Drop quarantined tests (file contains `cele2e-ci-unsafe`) from broad runs so
  // they never turn a full run red. Skipped by default; bypassed only when the
  // user explicitly named tests (`patterns.length > 0`) or passed `--complete`.
  if (!flagComplete && patterns.length === 0) {
    const before = testFiles.length;
    testFiles = testFiles.filter((f) => {
      try {
        return !readFileSync(f, 'utf-8').includes('cele2e-ci-unsafe');
      } catch {
        return true;
      }
    });
    const skipped = before - testFiles.length;
    if (skipped > 0) {
      console.log(
        `${dim}skipped ${skipped} quarantined (cele2e-ci-unsafe) test(s) — pass --complete or name one to run it${reset}`,
      );
    }
  }

  if (testFiles.length === 0) {
    console.error(`${red}No test files matched${reset}`);
    process.exit(1);
  }

  // e2e-confidence #258: optionally randomize order to surface order-dependent
  // bugs (the install-sh DNS bleed only manifested in a specific order). Seeded
  // + logged so a failing order replays exactly via --seed=<n>.
  if (flagShuffle) {
    const seed = (seedArg !== undefined && Number.isFinite(seedArg) ? seedArg : Date.now()) >>> 0;
    testFiles = seededShuffle(testFiles, seed);
    console.log(
      `${dim}--shuffle: randomized test order (seed=${seed}) — replay this exact order with --seed=${seed}${reset}`,
    );
  }

  // --keep keeps a single test network alive after the run; only one network
  // can live in `.e2e-persistent.json` at a time, so multiple tests would
  // either fight for that slot or run with no functional --keep at all. Bail
  // early with a clear message instead of silently doing the wrong thing.
  if (flagKeep && testFiles.length > 1) {
    console.error(
      `${red}--keep matched ${testFiles.length} tests; --keep only works with a single test.${reset}`,
    );
    console.error(
      `${dim}Narrow the pattern (e.g. \`cele2e run ${basename(testFiles[0], '.test.ts')} --keep\`) or drop --keep.${reset}`,
    );
    process.exit(1);
  }

  const history = loadTiming();
  let blockTiming = loadBlockTiming();
  const testNames = testFiles.map((f) => basename(f, '.test.ts'));
  const hasHistory = testNames.some((n) => history[n]);
  const suiteExpected = testNames.reduce((sum, n) => sum + (history[n] || 0), 0);

  const moduleLabel =
    moduleDirs.length === 1
      ? ` — ${basename(moduleDirs[0])}`
      : moduleDirs.length > 1
        ? ` — ${moduleDirs.length} modules`
        : '';
  console.log();
  console.log(`${bold}╔══════════════════════════════════════════════════════════════${reset}`);
  console.log(`${bold}║  Celilo E2E Tests${moduleLabel}${reset}`);
  if (hasHistory) {
    console.log(
      `${bold}║  ${testFiles.length} test(s) — expected ~${formatDuration(suiteExpected)}${reset}`,
    );
  } else {
    console.log(`${bold}║  ${testFiles.length} test(s) — first run, no timing history${reset}`);
  }
  console.log(`${bold}╚══════════════════════════════════════════════════════════════${reset}`);
  console.log();

  const suiteStart = Date.now();
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
  const resultsDir = join(testDir, 'results', timestamp);
  mkdirSync(resultsDir, { recursive: true });

  // runId scopes every event in this suite invocation. Read from env to
  // let callers (CI) inject a known id and subscribe to it before launching
  // cele2e; otherwise generate one and print it so the operator can correlate.
  const runId = process.env.CELE2E_RUN_ID || crypto.randomUUID();
  if (!process.env.CELE2E_RUN_ID) {
    console.log(`${dim}runId: ${runId}${reset}`);
  }
  // Print the results dir HERE, not only at the end: a run that dies mid-way
  // still wrote logs there, and `ls -t results | head -1` is not a safe way to
  // find them — after a refused start it returns the PREVIOUS run's numbers,
  // which look exactly like a clean pass. `cele2e last` reads the same record.
  console.log(`${dim}results:  ${resultsDir}${reset}`);
  writeLastRun({
    runId,
    resultsDir,
    startedAt: new Date(suiteStart).toISOString(),
    status: 'running',
    total: testFiles.length,
    passed: 0,
    failed: 0,
    skipped: 0,
    durationMs: 0,
  });
  emitRunStarted({
    runId,
    scenario: testFiles.length > 1 ? 'multi-test' : 'single-test',
    testNames,
    moduleDirs,
    startedAt: suiteStart,
  });

  // 'auto' resolves to 'render' when isTTY is true (operator at a
  // terminal sees animated spinners) and 'protocol' when isTTY is
  // false (a backgrounded run / any redirected stdout gets ASCII
  // `[progress:*]` markers — grep-friendly, no braille noise).
  // Prior to this change the mode was hardcoded 'render', so non-TTY
  // callers got `⣾ phase started` / `✔ phase done` lines per phase
  // (e.g. ~29 braille codepoints in a typical test log) — cosmetic
  // but actively unhelpful for log scanning and tooling.
  //
  // CI mode forces 'render' with isTTY:false (non-interactive render):
  // one ✔/✗ line per step, sub-events emitted inline, NO spinner and
  // NO `[progress:*]` markers. We can't rely on 'auto' here because
  // Forgejo's PTY makes isTTY truthy — which would pick the animated
  // footer and spam the log with one braille line per spinner tick.
  const displayMode: DisplayMode = ci ? 'render' : 'auto';
  const display = new ProgressDisplay({
    mode: displayMode,
    out: { write: process.stdout.write.bind(process.stdout), isTTY: interactive },
  });
  let passed = 0;
  let failed = 0;
  const results: TestResult[] = [];

  let interrupted = false;
  process.on('SIGINT', () => {
    if (interrupted) process.exit(130);
    interrupted = true;
    console.log(`\n${red}Interrupted. Stopping shared infrastructure...${reset}`);
    stopSharedInfra();
    saveTiming(history);
    saveBlockTiming(blockTiming);
    emitRunFailed({
      runId,
      durationMs: Date.now() - suiteStart,
      error: 'Interrupted by SIGINT',
    });
    process.exit(130);
  });

  for (let i = 0; i < testFiles.length; i++) {
    if (interrupted) break;

    const file = testFiles[i];
    const name = testNames[i];
    const logDir = join(resultsDir, name);
    mkdirSync(logDir, { recursive: true });

    const expected = history[name];

    // Remaining ETA from history
    const remainingExpected = testNames.slice(i).reduce((sum, n) => sum + (history[n] || 0), 0);
    const hasRemaining = testNames.slice(i).some((n) => history[n]);

    let header = `${bold}[${i + 1}/${testFiles.length}] ${name}${reset}`;
    if (expected) {
      header += ` ${dim}(~${formatDuration(expected)})${reset}`;
    }
    if (hasRemaining && i > 0) {
      header += ` ${dim}— ~${formatDuration(remainingExpected)} remaining${reset}`;
    }
    console.log(header);

    emitTestStarted({ runId, name, expectedDurationS: expected });
    const result = await runTest(file, name, expected, logDir, display);

    // e2e-confidence #254: name a test that leaves shared infra polluted, at the
    // boundary — before a later test fails mysteriously downstream. detect…()
    // also heals (re-scrubs), so a passing-but-polluting test is failed here AND
    // the next test still starts clean.
    const pollution = detectSharedDnsPollution();
    if (pollution && result.status === 'pass') {
      result.status = 'fail';
      result.error = `cross-test pollution (#254): this test left shared DNS dirty — ${pollution}`;
    }

    enforceBlockCaps(result, name, logDir, file);

    emitTestCompleted({
      runId,
      name,
      status:
        result.status === 'pass' ? 'pass' : result.status === 'suspicious' ? 'suspicious' : 'fail',
      durationMs: result.duration * 1000,
      error: result.error,
      logDir,
    });
    results.push(result);
    // Only save meaningful durations — crashes (<10s) and debug sessions
    // would poison future ETAs
    if (result.duration >= 10 && !result.hadDebugPause) {
      history[name] = result.duration;
    }
    // Recorded even for a short or failed run: a block that BLEW its cap is
    // exactly the measurement this file exists for, and a debug pause inflates
    // the suite total without touching the blocks that ran before it.
    if (result.blocks) {
      blockTiming = mergeBlockTiming(blockTiming, name, result.blocks);
      reportTightBlocks(name, file, result.blocks);
    }

    // Write persistent network info if --keep was set and we captured a project
    if (flagKeep && result.projectName) {
      writeFileSync(
        persistentFile,
        `${JSON.stringify(
          {
            projectName: result.projectName,
            testFile: file,
            createdAt: new Date().toISOString(),
          },
          null,
          2,
        )}\n`,
      );
      console.log(`${dim}  Persistent network: ${result.projectName}${reset}`);
      console.log(`${dim}  Reuse: ${basename(process.argv[1])} --reuse${reset}`);
      console.log(`${dim}  Tear down: cele2e down${reset}`);
      // The kept network outlives this process — hold the lock in a `kept`
      // state so the next run refuses to clobber it (cleared by `cele2e down`).
      markKept();
    }

    if (result.status === 'pass') {
      console.log(`  ${green}✔ passed (${result.duration}s)${reset}`);
      passed++;
    } else if (result.status === 'suspicious') {
      console.log(`  ${red}✗ suspicious pass — 0 assertions (${result.duration}s)${reset}`);
      console.log(`  ${red}  └─ ${result.error}${reset}`);
      failed++;
    } else {
      console.log(`  ${red}✗ failed (${result.duration}s)${reset}${stageSuffix(result.stages)}`);
      if (result.error) {
        console.log(`  ${red}  └─ ${result.error}${reset}`);
      }
      if (result.rawTail) {
        console.log();
        console.log(`${dim}  ── test output ──────────────────────────────────────${reset}`);
        for (const line of result.rawTail.split('\n')) {
          console.log(`  ${dim}${line}${reset}`);
        }
        console.log(`${dim}  ─────────────────────────────────────────────────────${reset}`);
      }
      failed++;
    }
    console.log();

    // The live-stack guard refused (celilo#1297): another run's stack is in
    // the way. Stop the whole run with exit 3 — the same code the lock
    // refusal uses — instead of grinding every remaining suite into the same
    // wall. Nothing of ours is running, so there is nothing to tear down.
    if (result.refused) {
      console.error(
        `\n${red}✗ startup cleanup refused — a live e2e stack is in the way:${reset}\n`,
      );
      if (result.rawTail) console.error(`${dim}${result.rawTail}${reset}\n`);
      console.error(`${dim}Clear the live stack with: cele2e down${reset}\n`);
      emitRunFailed({
        runId,
        durationMs: Date.now() - suiteStart,
        error: result.error ?? 'startup cleanup refused: live e2e stack in the way',
      });
      process.exit(3);
    }

    writeFileSync(
      join(logDir, 'result.json'),
      JSON.stringify({ status: result.status, duration: result.duration, error: result.error }),
    );

    try {
      execSync('docker network prune -f', { stdio: 'pipe', timeout: 10_000 });
    } catch {}
  }

  saveTiming(history);
  saveBlockTiming(blockTiming);

  // A --keep run keeps the WHOLE stack: the per-test project (persistent file
  // written in the loop) and the shared infra here. Tearing the shared pieces
  // down under --keep gave a kept stack with no name resolution and no certs
  // (celilo#1313) — half a system, in the mode the debugging method
  // prescribes. The exit stays one command: `cele2e down` sweeps every
  // celilo-e2e-* container, network and volume by name filter (bin/e2e-down)
  // and releases the kept lock, so the kept shared infra costs nothing extra
  // to reap. markKept() here too: with shared infra left up, a run that
  // captured no project still leaves a live stack, and an unmarked lock makes
  // the next run's live-stack refusal read as contention instead of a
  // deliberate kept stack. Idempotent with the per-test markKept().
  if (flagKeep) {
    markKept();
    console.log(`${dim}Keeping shared infrastructure (DNS, CA, registry) for debugging.${reset}`);
    console.log(`${dim}  Tear down everything (project + shared infra): cele2e down${reset}`);
  } else {
    console.log(`${dim}Stopping shared infrastructure...${reset}`);
    stopSharedInfra();
  }

  const suiteDuration = Math.floor((Date.now() - suiteStart) / 1000);

  // Stages a failing earlier stage blocked are reported separately from real
  // failures. Counting them together turns one bad fixture line into "9 failed"
  // and buries the single defect that actually exists.
  const stageSkipped = results.reduce((sum, r) => sum + (r.stages?.skipped ?? 0), 0);

  console.log();
  console.log(`${bold}╔══════════════════════════════════════════════════════════════${reset}`);
  console.log(
    `${bold}║  Results: ${green}${passed} passed${reset}${bold}, ${red}${failed} failed${reset}${bold} — ${formatDuration(suiteDuration)} total${reset}`,
  );
  if (stageSkipped > 0) {
    console.log(
      `${bold}║${reset}  ${dim}${stageSkipped} stage(s) skipped — blocked by an earlier stage, not separate defects${reset}`,
    );
  }
  console.log(`${bold}╠══════════════════════════════════════════════════════════════${reset}`);

  for (const r of results) {
    const icon = r.status === 'pass' ? '✓' : '✗';
    const color = r.status === 'pass' ? green : red;
    console.log(
      `${bold}║${reset}  ${color}${icon}${reset} ${r.name.padEnd(28)} ${dim}${(`${r.duration}s`).padStart(6)}${reset}${stageSuffix(r.stages)}`,
    );
    if (r.error) {
      console.log(`${bold}║${reset}    ${red}└─ ${r.error}${reset}`);
    }
  }

  console.log(`${bold}╚══════════════════════════════════════════════════════════════${reset}`);
  console.log();
  console.log(`${dim}Full logs: ${resultsDir}${reset}`);
  console.log(`${dim}Machine-readable: cele2e last --json${reset}`);

  writeFileSync(
    join(resultsDir, 'summary.json'),
    JSON.stringify(
      {
        timestamp,
        total: testFiles.length,
        passed,
        failed,
        stageSkipped,
        duration: suiteDuration,
      },
      null,
      2,
    ),
  );

  writeLastRun({
    runId,
    resultsDir,
    startedAt: new Date(suiteStart).toISOString(),
    status: failed > 0 ? 'failed' : 'completed',
    total: testFiles.length,
    passed,
    failed,
    skipped: stageSkipped,
    durationMs: suiteDuration * 1000,
  });

  const gitignore = join(testDir, '.gitignore');
  const content = existsSync(gitignore) ? readFileSync(gitignore, 'utf-8') : '';
  const adds: string[] = [];
  if (!content.includes('results/')) adds.push('results/');
  if (!content.includes('.e2e-timing.json')) adds.push('.e2e-timing.json');
  if (!content.includes('.e2e-block-timing.json')) adds.push('.e2e-block-timing.json');
  if (adds.length) writeFileSync(gitignore, `${content.trimEnd()}\n${adds.join('\n')}\n`);

  emitRunCompleted({
    runId,
    total: testFiles.length,
    passed,
    failed,
    durationMs: suiteDuration * 1000,
    resultsDir,
  });

  notify('cele2e', `${passed} passed, ${failed} failed — ${formatDuration(suiteDuration)}`);
  process.exit(failed > 0 ? 1 : 0);
}

// Ambient runId so the top-level catch can emit run-failed even when
// main() throws before it allocates one. main() may overwrite this with
// a caller-supplied CELE2E_RUN_ID to keep the value consistent.
const ambientRunId = process.env.CELE2E_RUN_ID || crypto.randomUUID();
const ambientStart = Date.now();
process.env.CELE2E_RUN_ID = ambientRunId;

main().catch((err) => {
  emitRunFailed({
    runId: ambientRunId,
    durationMs: Date.now() - ambientStart,
    error: err instanceof Error ? err.message : String(err),
  });
  console.error(err);
  notify('cele2e', 'run failed');
  process.exit(1);
});