#!/usr/bin/env bun /** * cele2e — E2E test CLI for Celilo-deployed applications. * * Usage: * cele2e run [pattern] [--keep] [--reuse] [--live] [--published] * cele2e up [--caddy|--full-stack|--infrastructure|--custom ''] * cele2e down [--keep] [--all] * cele2e build-infra [module-dir...] [--save] * cele2e clear-timing * cele2e version * cele2e shell [container] * cele2e status * cele2e doctor * cele2e host * cele2e last [--json] * cele2e load * cele2e scaffold */ import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync, rmSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { parse as parseYaml } from 'yaml'; import { diagnose, formatHolderLine, formatReport } from '../doctor'; import { readLastRun } from '../last-run'; import { E2eBusyError, type LockHolder, acquireRunLock, clearLock, lockStatus, markKept, } from '../run-lock'; import { runBuild } from './build'; import { generateBashCompletion, generateZshCompletion, getCompletions } from './completion'; import { runHost } from './host'; import { discoverSuiteTests, findAllModules, resolvePatternTarget, resolveRunTargets, } from './module-discovery'; import { runScaffold } from './scaffold'; const PKG_DIR = resolve(import.meta.dir, '../..'); const BIN_DIR = join(PKG_DIR, 'bin'); function readManifestTestsDir(manifestPath: string): string | undefined { try { const parsed = parseYaml(readFileSync(manifestPath, 'utf-8')) as { e2e?: { tests_dir?: string }; }; return parsed?.e2e?.tests_dir; } catch { return undefined; } } function detectTestDir(): { stateDir: string; testsPath: string } { if (process.env.E2E_TEST_DIR) { const stateDir = process.env.E2E_TEST_DIR; return { stateDir, testsPath: join(stateDir, 'tests') }; } const cwd = process.cwd(); const manifestPath = join(cwd, 'manifest.yml'); if (existsSync(manifestPath)) { const testsDir = readManifestTestsDir(manifestPath); if (testsDir) { const testsPath = resolve(cwd, testsDir); return { stateDir: dirname(testsPath), testsPath }; } } if (existsSync(join(cwd, 'tests'))) return { stateDir: cwd, testsPath: join(cwd, 'tests') }; // The repo's own top-level suite lives at e2e/tests/, which is what `list` // enumerates. Without this, `cele2e list` shows a test that `cele2e run // ` then cannot find — the run/list disagreement in #412. if (existsSync(join(cwd, 'e2e', 'tests'))) { return { stateDir: join(cwd, 'e2e'), testsPath: join(cwd, 'e2e', 'tests') }; } return { stateDir: cwd, testsPath: join(cwd, 'tests') }; } /** * Resolve the repo root to scan for the full e2e suite (`run --all` / `list`). * * The workspace `./cele2e` wrapper cd's into packages/e2e before exec'ing, so * process.cwd() points at the e2e package — useless for module discovery. The * wrapper exports E2E_INVOKE_CWD with the operator's real PWD; we walk up from * there (or process.cwd() as a fallback) to the nearest ancestor that has a * `modules/` directory, which is the infra repo root. This is what makes the * full-regression command immune to the wrapper's internal cd (ISS-0013). */ function resolveSuiteRoot(): string { const start = resolve(process.env.E2E_INVOKE_CWD || process.cwd()); let dir = start; for (;;) { if (existsSync(join(dir, 'modules'))) return dir; const parent = dirname(dir); if (parent === dir) break; dir = parent; } return start; } /** * Say out loud when we cleared this session's own kept stack. Silence would be * worse than the old refusal: the operator asked for a kept stack, and it is * gone now for a reason they should be able to read. */ function noteAutoRelease(holder: LockHolder | null): void { if (!holder) return; console.log( `Auto-released your own kept stack (${holder.test}, held since ${holder.startedAt}) and continuing.`, ); console.log('Use `cele2e run --reuse` instead if you meant to run against it.'); } function runScript(script: string, args: string[], env?: Record): never { const result = spawnSync(join(BIN_DIR, script), args, { stdio: 'inherit', env: { ...process.env, ...env }, }); process.exit(result.status ?? 1); } function printHelp(): void { console.log(` cele2e — E2E test CLI for Celilo-deployed applications Usage: cele2e [options] Commands: run [pattern] Run e2e tests (pattern filters by test file name) run --all Run the FULL regression: every module suite + top-level e2e/tests/ list List every discoverable Docker e2e test (module + top-level) up [preset] Start interactive test network down Tear down running network build-infra [dirs...] Rebuild Docker images and package all modules as .netapps clear-timing Clear saved timing history used for test ETAs version Show cele2e version shell [container] Shell into a running container (default: management) status Show run-lock holder + network status (exit 3 if busy) doctor Check the environment a run needs (run's implicit preflight) host Bring the Docker host VM up/down in the shape the rig needs last Show the most recent run's results dir + counts release Free a run-lock left by --keep/up (does not tear down) load Load cached Docker images from tarball scaffold Generate a new test file from template completion Generate shell completion script (zsh|bash) Options for \`run\`: --all Run every module suite AND the top-level e2e/tests/ (full regression) --all-modules Run every module suite (no top-level tests) --keep Keep network running after tests --reuse Reuse existing network if running --live Use live (non-simulated) internet --published Use published .netapp packages --notify Desktop notification when the run finishes (best-effort) --source-cli Run the celilo CLI from the mounted workspace instead of the one baked into the management image. Roughly doubles each celilo command's start-up; iterate without a \`build-infra\` --ci Plain log output: one ✔/✗ line per step, no spinner (auto-on when a CI env var is set; --no-ci forces off) Options for \`up\`: --caddy Start with caddy machine in DMZ --full-stack Start with caddy, IDP, and DB machines --custom Custom machine spec JSON Verbs for \`host\` (macOS/colima; a no-op where docker runs natively): status Report the VM against the policy (exit 1 if out of policy) up Start it, or say exactly why a running one is out of policy down Stop it (refuses while the e2e run-lock is held) reset --yes Recreate it — the ONLY way to change the mount transport. DESTROYS every built image; budget one \`cele2e build-infra\` Options for \`down\`: --keep Stop containers but preserve volumes --all Also stop competing module containers Options for \`doctor\` / \`last\` / \`status\`: --json Machine-readable output Options for \`build-infra\`: --save Save Docker images to tarball after build --skip-modules Skip .netapp packaging (only rebuild Docker images) --published Bake management:latest with the PUBLISHED @celilo/cli from real npm (default: bake the monorepo DEV cli via the sim) Examples: cele2e run --all # FULL regression: every module suite + top-level e2e/tests/ cele2e list # enumerate every discoverable Docker e2e test cele2e run # in a module: run its tests; outside: run all modules' tests cele2e run deploy # run tests matching "deploy" cele2e run caddy-direct-internet # from modules/, finds the matching module automatically cele2e run --reuse deploy # reuse existing network cele2e up --full-stack # interactive full-stack environment cele2e shell caddy # debug running caddy container cele2e build-infra # rebuild Docker images + repackage all modules cele2e build-infra --skip-modules # rebuild Docker images only (faster) cele2e host status # is the Docker host VM shaped for the rig? cele2e run caddy-uninstall --source-cli # run the WORKSPACE cli, not the baked one cele2e scaffold lunacycle-auth # create tests/lunacycle-auth.test.ts cele2e completion zsh # print zsh completion script `); } const [, , command, ...args] = process.argv; const { stateDir, testsPath: detectedTestsPath } = detectTestDir(); // Handle --get-completions before routing (used by shell completion scripts) if (command === '--get-completions') { const words = args.slice(0, -1); const current = Number.parseInt(args[args.length - 1] || '0', 10); const suggestions = getCompletions(words, current); process.stdout.write(`${suggestions.join('\n')}\n`); process.exit(0); } switch (command) { case 'run': { // Full-suite modes (ISS-0013). `--all` runs every module suite AND the // repo's top-level e2e/tests/ in one invocation; `--all-modules` runs just // the module suites. Both resolve the repo root via resolveSuiteRoot() so // they work through the `./cele2e` wrapper's cd into packages/e2e. if ( args.includes('--all') || args.includes('--all-modules') || args.includes('--complete') || args.includes('--ci-safe') ) { // --all / --complete / --ci-safe all run the full discovery (every module + // the repo's top-level e2e/tests/). Quarantined (`cele2e-ci-unsafe`) tests // are skipped by default; only --complete opts back into them (the runner // sees the flag — it's kept below). --all-modules stays modules-only. const wantTopLevel = args.includes('--all') || args.includes('--complete') || args.includes('--ci-safe'); const suiteRoot = resolveSuiteRoot(); const allModules = findAllModules(suiteRoot); const env: Record = { E2E_TEST_DIR: join(suiteRoot, 'e2e'), E2E_MODULE_DIRS: allModules.join('\n'), }; const topLevel = join(suiteRoot, 'e2e', 'tests'); if (wantTopLevel && existsSync(topLevel)) { env.E2E_TOP_LEVEL_TESTS = topLevel; } if (allModules.length === 0 && !env.E2E_TOP_LEVEL_TESTS) { console.error(`No e2e tests found under ${suiteRoot}`); console.error('Run from the infra repo root (or set E2E_INVOKE_CWD to it).'); process.exit(1); } runScript( 'e2e-run', // Strip the discovery-only aliases; KEEP --complete (the runner uses it to // opt back into quarantined tests) and --ci-safe (harmless/no-op now). args.filter((a) => a !== '--all' && a !== '--all-modules'), env, ); } // Check if any non-flag arg is a module path (has manifest.yml) const modulePath = args .filter((a) => !a.startsWith('--')) .find((a) => existsSync(join(resolve(a), 'manifest.yml'))); if (modulePath) { const absModulePath = resolve(modulePath); runScript( 'e2e-run', args.filter((a) => a !== modulePath), { E2E_TEST_DIR: stateDir, E2E_MODULE_DIR: absModulePath, }, ); } // If cwd is not a module, walk sibling modules so the user can invoke // `cele2e run` from infra/modules/ (or the repo root) without cd-ing in. const cwdHasManifest = existsSync(join(process.cwd(), 'manifest.yml')); const positionals = args.filter((a) => !a.startsWith('--')); if (!cwdHasManifest) { if (positionals.length === 1) { // Resolve the pattern against the FULL discovered suite (every module + // the repo's top-level e2e/tests/), rooted at resolveSuiteRoot() like // `list` and `--all`. The wrapper cd's into packages/e2e before // exec'ing, so the old walk down from process.cwd() started below the // repo root: a test name `cele2e list` printed came back "No test // files matched" from `cele2e run `. const target = resolvePatternTarget(resolveSuiteRoot(), positionals[0]); if (target?.kind === 'top-level') { runScript('e2e-run', args, { E2E_TEST_DIR: target.dir, E2E_TESTS_PATH: target.testsPath, }); } else if (target?.kind === 'module') { runScript('e2e-run', args, { E2E_TEST_DIR: stateDir, E2E_MODULE_DIR: target.moduleDir, }); } else if (target?.kind === 'ambiguous') { console.error(`Multiple tests match "${positionals[0]}":`); for (const m of target.matches) { console.error(` ${m.group}: ${m.name}`); } console.error('\nDisambiguate with a longer pattern (an exact name always wins).'); process.exit(1); } } else if (positionals.length === 0) { const allModules = findAllModules(process.cwd()); if (allModules.length > 0) { runScript('e2e-run', args, { E2E_TEST_DIR: stateDir, E2E_MODULE_DIRS: allModules.join('\n'), }); } } else { // Several names: resolve each independently against the full suite // and feed the runner's multi-module env (E2E_MODULE_DIRS / // E2E_TOP_LEVEL_TESTS). The old fall-through ran top-level discovery // only, so module suites reported "No test files matched" — which // reads as a typo, not a limitation. const resolved = resolveRunTargets(resolveSuiteRoot(), positionals); if (resolved.kind === 'no-match') { console.error(`No suite matches "${resolved.pattern}". List them: cele2e list`); process.exit(1); } if (resolved.kind === 'ambiguous') { console.error(`Multiple tests match "${resolved.pattern}":`); for (const m of resolved.matches) { console.error(` ${m.group}: ${m.name}`); } console.error('\nDisambiguate with a longer pattern (an exact name always wins).'); process.exit(1); } const env: Record = { E2E_TEST_DIR: stateDir }; if (resolved.modules.length > 0) { env.E2E_MODULE_DIRS = resolved.modules.join('\n'); } if (resolved.topLevel) { env.E2E_TOP_LEVEL_TESTS = join(resolveSuiteRoot(), 'e2e', 'tests'); } runScript('e2e-run', args, env); } } runScript('e2e-run', args, { E2E_TEST_DIR: stateDir, E2E_TESTS_PATH: detectedTestsPath }); break; } case 'list': { // Enumerate every discoverable Docker e2e test (module + top-level) so the // suite is inspectable without reading runner.ts (ISS-0013). const suiteRoot = resolveSuiteRoot(); const tests = discoverSuiteTests(suiteRoot); if (tests.length === 0) { console.error(`No e2e tests found under ${suiteRoot}`); console.error('Run from the infra repo root (or set E2E_INVOKE_CWD to it).'); process.exit(1); } const wantPaths = args.includes('--paths'); let lastGroup = ''; for (const t of tests) { if (t.group !== lastGroup) { console.log(`\n${t.group}:`); lastGroup = t.group; } console.log(wantPaths ? ` ${t.name} ${t.file}` : ` ${t.name}`); } console.log(`\n${tests.length} tests. Run all: cele2e run --all`); process.exit(0); break; } case 'up': { // `up` leaves an interactive network running, so it holds the run-lock in a // kept state (cleared by `cele2e down`/`release`) to stop a concurrent run // from wiping it. try { const outcome = acquireRunLock({ test: 'up', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID(), }); noteAutoRelease(outcome.autoReleasedOwnKept); markKept(); } catch (err) { if (err instanceof E2eBusyError) { console.error(err.message); process.exit(3); } throw err; } runScript('e2e-up', args); break; } case 'down': // Tearing the network down frees the (possibly kept) lock. clearLock(); runScript('e2e-down', args, { E2E_TEST_DIR: stateDir }); break; case 'shell': runScript('e2e-shell', args); break; case 'release': { // Free a lock left by `--keep`/`up` (a `kept` stack) so the next run can // proceed. Does not tear the stack down — use `cele2e down` for that. const had = clearLock(); console.log(had ? 'Released the e2e run-lock.' : 'No e2e run-lock held.'); process.exit(0); break; } case 'status': { // Run-lock is the contention surface other sessions poll. Report it first; // exit 0 = free, 3 = busy. `--json` emits a machine-readable snapshot. const lock = lockStatus(); if (args.includes('--json')) { console.log(JSON.stringify(lock)); process.exit(lock.free ? 0 : 3); } console.log('=== Run Lock ==='); if (lock.free) { console.log(' free'); } else if (lock.holder) { // Heartbeat age is always printed: a wedged holder keeps a live PID, so // the only thing separating "slow" from "hung for 32 minutes" is how long // ago it last beat — previously visible only by reading beatAt by hand. console.log(` BUSY — ${formatHolderLine(lock.holder)}`); if (lock.suspect) { console.log( ' ⚠ SUSPECT: the process is alive but has stopped beating. It is probably wedged;', ); console.log(` inspect or kill pid ${lock.holder.pid}, then \`cele2e down\`.`); } if (lock.ownKept) { console.log(' This is YOUR OWN kept stack — the next `cele2e run` auto-releases it.'); } } console.log(''); const persistentFile = join(stateDir, '.e2e-persistent.json'); let projectName: string | undefined; if (existsSync(persistentFile)) { try { const data = JSON.parse(readFileSync(persistentFile, 'utf-8')); projectName = data.projectName; console.log('=== Persistent Test Network ==='); console.log(` Project: ${data.projectName}`); console.log(` Test: ${data.testFile}`); console.log(` Created: ${data.createdAt}`); console.log(' Reuse: cele2e run --reuse'); console.log(' Tear down: cele2e down'); console.log(''); } catch { /* ignore parse errors */ } } runScript('e2e-status', args, projectName ? { CELILO_E2E_PROJECT: projectName } : undefined); break; } case 'doctor': { // The single "is my environment sane?" answer. `run` calls the same // diagnose() as its preflight, so what doctor reports is exactly what a run // will refuse on — no second, drifting copy of the rules. const report = diagnose({ pkgDir: PKG_DIR }); if (args.includes('--json')) { console.log(JSON.stringify(report, null, 2)); process.exit(report.ok ? 0 : 1); } console.log('=== cele2e doctor ==='); for (const line of formatReport(report)) console.log(line); console.log(''); console.log( report.ok ? 'Environment looks runnable.' : 'Environment is NOT runnable — see fixes above.', ); process.exit(report.ok ? 0 : 1); break; } case 'host': { // Deliberately outside the run-lock: `status` is read-only, and `up`/`down`/ // `reset` take the lock's VERDICT (see host.ts lockBlocks) rather than the // lock itself — taking it would make `cele2e host down` unable to report // that somebody else's run is what stopped it. process.exit(runHost(args)); break; } case 'last': { // Which results directory was MINE? Inferring it from `ls -t results` returns // the newest run, which after a refused start is somebody else's — a suite // that never executed then reads as a clean pass. const last = readLastRun(); if (args.includes('--json')) { console.log(JSON.stringify(last)); process.exit(last ? 0 : 1); } if (!last) { console.error('No cele2e run recorded yet.'); process.exit(1); } console.log(`runId: ${last.runId}`); console.log(`resultsDir: ${last.resultsDir}`); console.log(`status: ${last.status}`); console.log( `results: ${last.passed} passed, ${last.failed} failed${last.skipped > 0 ? `, ${last.skipped} skipped (blocked by an earlier stage)` : ''} of ${last.total}`, ); process.exit(0); break; } case 'load': runScript('e2e-load', args); break; case 'build-infra': { const save = args.includes('--save'); const skipModules = args.includes('--skip-modules'); const published = args.includes('--published'); const moduleDirs = args.filter((a) => !a.startsWith('--')); // build-infra mutates the shared Docker images — serialize against runs. // Released via the lock's exit handler (runBuild may process.exit on failure). try { const outcome = acquireRunLock({ test: 'build-infra', runId: process.env.CELE2E_RUN_ID ?? crypto.randomUUID(), }); noteAutoRelease(outcome.autoReleasedOwnKept); } catch (err) { if (err instanceof E2eBusyError) { console.error(err.message); process.exit(3); } throw err; } await runBuild({ pkgDir: PKG_DIR, moduleDirs, save, skipModules, published }); break; } case 'clear-timing': { const timingFile = join(stateDir, '.e2e-timing.json'); if (existsSync(timingFile)) { rmSync(timingFile); console.log(`Cleared timing data: ${timingFile}`); } else { console.log(`No timing data found at ${timingFile}`); } break; } case 'scaffold': { const name = args[0]; if (!name) { console.error('Error: scaffold requires a test name\n cele2e scaffold '); process.exit(1); } runScaffold({ testsPath: detectedTestsPath, name }); break; } case 'version': { const pkg = await import('../../package.json', { with: { type: 'json' } }); console.log(`cele2e v${pkg.default.version}`); break; } case 'completion': { const shell = args[0]; if (shell === 'zsh') { process.stdout.write(generateZshCompletion()); } else if (shell === 'bash') { process.stdout.write(generateBashCompletion()); } else { console.error('Usage: cele2e completion '); process.exit(1); } break; } case undefined: case '--help': case '-h': case 'help': printHelp(); break; default: console.error(`Unknown command: ${command}\n`); printHelp(); process.exit(1); }