// The e2e app's PROCESS half: spawn it on a free port, wait for `/readyz`, restart it on the same // port as a deploy, stop it. Split from `e2e-app.ts`, which owns the DATABASE half (the throwaway // state directory, the reset and the seed), so this half runs against any root with an entry — // which is what lets a unit test drive it without a Postgres. // why: the readiness poll must not go through `globalThis.fetch` — inside `bun test` the testing // preload SEALS it, and the app's `/readyz` would be refused as egress. `node:http` is not sealed. import { get } from 'node:http'; // why: Bun exposes no path API — the CLI's bin is joined onto its package directory. import { dirname, join } from 'node:path'; import { assert, DRAIN_DEADLINE_DEFAULT_MS, READINESS_GRACE_DEFAULT_MS } from '@ultimat3/core'; import { E2eAppFailedError } from './e2e-errors'; import { freePort, freePortApartFrom } from './free-port'; /** `x dev` (sync included), or the production entry `apps/web/server.ts` under `ROLE=web`. */ export type E2eAppMode = 'dev' | 'serve'; /** * The `x` the APP installed, read off `@ultimat3/cli`'s own `bin` entry and resolved from the app * root — never a global `x`, which may be a different version of the framework, and never beside * this module: `@ultimat3/testing` does not depend on the CLI, and the `import.meta.dir/bin.ts` * this was while the driver lived in cli named a file that does not exist once it moved here. */ export async function xBin(root: string): Promise { let manifest: string; try { manifest = Bun.resolveSync('@ultimat3/cli/package.json', root); } catch { throw new E2eAppFailedError({ step: 'resolve @ultimat3/cli', output: `@ultimat3/cli does not resolve from ${root}`, fix: 'bun add -d @ultimat3/cli # in the app root: the e2e app is spawned through the CLI the app itself installed', }); } const declared: unknown = await Bun.file(manifest).json(); const bin = typeof declared === 'object' && declared !== null ? Reflect.get(declared, 'bin') : undefined; const x = typeof bin === 'object' && bin !== null ? Reflect.get(bin, 'x') : undefined; if (typeof x !== 'string') { throw refuse('resolve @ultimat3/cli', `${manifest} declares no "bin": { "x": … } entry`); } return join(dirname(manifest), x); } const POLL_MS = 250; /** * How long one `/readyz` probe may hang before it counts as "not yet": an app that accepted the * connection mid-boot and has not answered. The readiness deadline can overshoot by one of these. */ export const READY_PROBE_MS = POLL_MS * 4; /** * What a SIGTERM'd app is given before it is killed outright: its own designed drain at the * defaults — the readiness grace a deployed app holds, then `drain.deadlineMs` — because a stop * shorter than the drain kills an app that was doing what it was told. */ const TERM_GRACE_MS = READINESS_GRACE_DEFAULT_MS + DRAIN_DEADLINE_DEFAULT_MS; /** After SIGKILL: the kernel's to deliver, bounded for a container whose init never reaps. */ const KILL_GRACE_MS = 5_000; /** * The longest `stop()` is DESIGNED to take: the drain, then the kill. THE deadline a hook that * stops an e2e app derives from — unbounded, a wedged app held the hook until Bun killed it, and * the browser close behind it in the same hook never ran. */ export const E2E_APP_STOP_MS = TERM_GRACE_MS + KILL_GRACE_MS; export interface SpawnedE2eApp { /** `http://localhost:`, no trailing slash. */ readonly base: string; /** The last of what the CURRENT process printed, stdout and stderr, bounded. */ log(): string; /** Kill the process. Idempotent. */ stop(): Promise; /** Kill it and start it again on the SAME port, with `env` added — a deploy. Refused after `stop()`. */ restart(env?: Readonly>): Promise; } export interface SpawnE2eAppOptions { readonly root: string; readonly mode: E2eAppMode; /** Every spawn's environment, on top of this process's minus `NODE_ENV`. */ readonly env: Readonly>; /** Already screened finite by the caller. */ readonly readyTimeoutMs: number; /** `xBin(root)`, when the caller already resolved it; resolved here otherwise, in `dev` mode only. */ readonly bin?: string | undefined; /** How long a SIGTERM'd app has before SIGKILL. Defaults to the app's own drain at its defaults. */ readonly termGraceMs?: number | undefined; } /** * This process's environment minus what makes the child a TEST process. Spawned from `bun test`, * `NODE_ENV=test` rode along and the app resolved its environment as `test` — so a development-only * seam (the demo viewer an app installs instead of a sign-in route) was off and every page 401'd. * The app under e2e is a development app unless the caller's `env` says otherwise. * * Minus the two bindings `ULTIMATE_STATE_DIR` relocates, too — the database and the disk. The * harness boots on a throwaway state directory, and a developer's exported `DATABASE_URL` (their * integration-test Postgres) turned that into a reset of a database the run does not own: `x db * reset` refused it, and the e2e step went red for an environment reason (#674). A run that WANTS * an external one passes it in `startE2eApp({ env })`, which is spread after this. */ export const inherited = (): Record => { const { NODE_ENV: _node, DATABASE_URL: _db, S3_ENDPOINT: _disk, ...rest } = Bun.env; return { ...rest, ULTIMATE_ENV: 'development' }; }; export const refuse = (step: string, output: string): E2eAppFailedError => new E2eAppFailedError({ step, output }); /** Spawn the app and answer once `/readyz` does; refuses with the app's own output tail. */ export async function spawnE2eApp(options: SpawnE2eAppOptions): Promise { const deadline = options.readyTimeoutMs; const port = freePort(); // Its own scrape port too: every role opens one, the default is a fixed 9090, and a second app — // or a developer's `x dev` — already holding it is an app that dies at boot with X_PORT_IN_USE. // Apart from the web port AND its neighbour, which `x dev` binds the sync node on: the OS may hand // a just-released port straight back (`freePortApartFrom`). const metricsPort = freePortApartFrom([port, port + 1]); const base = `http://localhost:${String(port)}`; const command = options.mode === 'serve' ? ['bun', 'apps/web/server.ts'] : ['bun', options.bin ?? (await xBin(options.root)), 'dev', '--port', String(port)]; const spawnApp = (extra: Readonly>) => Bun.spawn(command, { cwd: options.root, env: { ...inherited(), ...options.env, METRICS_PORT: String(metricsPort), // The origin the app is actually reachable at. A page that renders a typed client builds // its absolute URLs from it, and without it `/feed` answered 500 with X_ENV_MISSING APP_URL. APP_URL: base, ...(options.mode === 'serve' ? { ROLE: 'web', PORT: String(port) } : {}), ...extra, }, stdout: 'pipe', stderr: 'pipe', }); const termGraceMs = options.termGraceMs ?? TERM_GRACE_MS; // On the monotonic clock: counted in polls, every probe an app wedged mid-boot held for its // whole `READY_PROBE_MS` stretched a 90 s deadline to 450 s, past any hook derived from it. const ready = async (child: ReturnType, tail: () => string): Promise => { const until = performance.now() + deadline; while (performance.now() < until) { if (child.exitCode !== null) break; if (await answersOk(`${base}/readyz`)) return; await Bun.sleep(POLL_MS); } await end(child, termGraceMs); throw refuse(`${command.join(' ')} never answered ${base}/readyz`, tail()); }; let child = spawnApp({}); // Drained from the start and kept bounded: an app's log is unbounded, and a pipe nobody reads // fills its buffer and blocks the child on its next write — an app that "never got ready". let tail = drainTail(child.stdout, child.stderr); let stopped = false; const stop = async (): Promise => { if (stopped) return; stopped = true; await end(child, termGraceMs); }; try { await ready(child, tail); } catch (error) { await stop(); throw error; } return { base, log: () => tail(), stop, async restart(next: Readonly> = {}): Promise { // `stop()` is final. A child respawned here would be one no later `stop()` kills — the flag // already says done — and `startE2eApp` has deleted the state directory it would boot on. assert( !stopped, 'restart() was called on an e2e app that was already stopped, so there is no app to deploy over', 'startE2eApp({ root }) again for a fresh app — restart() is for an app that is still running', ); // The same port and the same state directory — a deploy, not a second app: a tab already // open on `base` sees the new build on its next request, and the data it wrote is still there. await end(child, termGraceMs); child = spawnApp(next); tail = drainTail(child.stdout, child.stderr); await ready(child, tail); }, }; } const within = (ms: number, work: Promise): Promise => Promise.race([work, Bun.sleep(ms)]); /** SIGTERM, the grace, then SIGKILL: bounded at `termGraceMs + KILL_GRACE_MS`, never forever. */ async function end(child: Bun.Subprocess, termGraceMs: number): Promise { child.kill(); const exited = (await within( termGraceMs, child.exited.then(() => true), )) === true; if (exited) return; child.kill('SIGKILL'); await within(KILL_GRACE_MS, child.exited); } const TAIL_CHARS = 16_000; /** Read both streams to their end in the background; answer the last `TAIL_CHARS` of either. */ function drainTail(...streams: readonly ReadableStream[]): () => string { let text = ''; for (const stream of streams) { void (async () => { const decoder = new TextDecoder(); for await (const chunk of stream) { text = (text + decoder.decode(chunk, { stream: true })).slice(-TAIL_CHARS); } })().catch(() => undefined); } return () => text; } /** One GET, answered as "2xx or not" — never a throw, and never through the sealed `fetch`. */ function answersOk(url: string): Promise { return new Promise((resolve) => { const request = get(url, (response) => { response.resume(); const status = response.statusCode ?? 0; resolve(status >= 200 && status < 300); }); request.on('error', () => resolve(false)); request.setTimeout(POLL_MS * 4, () => { request.destroy(); resolve(false); }); }); }