/** * cele2e doctor — "is my environment sane?" as one mechanical answer. * * Every check here exists because its absence was diagnosed as a product bug * first. They share a shape: the mistake is silent, and the failure surfaces * later and somewhere unrelated. A missing bake shows up as an SSH error * against a firewall IP; a pruned base image shows up as a TLS handshake * timeout 16 images into a build. Each one is cheap to detect BEFORE a run * starts, so `run` calls this as its implicit preflight and refuses rather * than starting something that cannot succeed. * * Docker access is behind DoctorProbe (Rule 2.3) so every check is unit- * testable with the condition deliberately broken — which is the point: a gate * nobody has seen fail is not a gate (Rule 7.6). */ import { execFileSync } from 'node:child_process'; import { existsSync, readFileSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; import { type HostFacts, type HostVmFacts, evaluateHostVm, readHostFacts, readHostVmFacts, recommendedBudget, } from './host-vm'; import { CONTAINER_PREFIX } from './proxmox-provisioner'; import { findMonorepoRoot } from './repo-root'; import { type LockHolder, type LockStatus, formatAge, heartbeatAgeMs, isSameSession, isSuspect, lockStatus, } from './run-lock'; import { PUBLISHED_FINGERPRINT_PREFIX, SOURCE_LABEL, computeSourceFingerprint, } from './source-fingerprint'; /** * Images the per-test compose references by `image:` alone — it has no `build:` * for them, so docker CANNOT produce them on demand. `cele2e build-infra` bakes * both (install.sh, then `docker commit`). Missing → the run is doomed. */ export const MANAGEMENT_LATEST = 'celilo-e2e/management:latest'; export const BAKED_MANAGEMENT_IMAGES = [ MANAGEMENT_LATEST, 'celilo-e2e/management:vanilla', ] as const; /** * The bake commits a container that was started with `sleep infinity`, so it * must override Cmd back to the management image's real entrypoint. When that * override is lost the container comes up without ever running /startup.sh, the * ssh-keys volume is never populated, and every machine the test adds fails * with "Cannot connect to root@ with provided SSH key" — an error that * reads as a network or firewall bug and is neither (ce-um6). */ export const EXPECTED_MANAGEMENT_CMD = '/startup.sh'; /** Warn above this much reclaimable image space — the pressure that makes people prune. */ const RECLAIMABLE_WARN_BYTES = 20 * 1024 ** 3; export type CheckStatus = 'ok' | 'warn' | 'fail'; export interface DoctorCheck { /** Stable machine-readable id, e.g. 'management-image'. */ name: string; status: CheckStatus; /** What was actually found. */ detail: string; /** The exact command that fixes it. Present whenever status is not 'ok'. */ remedy?: string; } export interface DoctorReport { checks: DoctorCheck[]; /** True when no check failed. Warnings do not block a run. */ ok: boolean; } /** * A leaked per-test compose stack: one timestamped-project network plus whatever * is still attached to it. The project name embeds the epoch-ms the suite * started, which is what makes the leaker attributable after the fact. */ export interface LeakedStack { /** e.g. `celilo-e2e-1788615872911_dmz` */ network: string; /** Attached container names, e.g. `celilo-e2e-lxc-104`. */ containers: string[]; } /** Every Docker fact doctor needs, injectable so the checks are testable. */ export interface DoctorProbe { /** Is this image present in the local store? An untagged ref means `:latest`. */ imageExists(ref: string): boolean; /** The image's configured Cmd, or null when the image is absent. */ imageCmd(ref: string): string[] | null; /** One label off the image, or null when the image or the label is absent. */ imageLabel(ref: string, label: string): string | null; /** * `celilo --version` run INSIDE the image, or null when the binary is absent * or does not run. This is the only check that opens the image up; every * other one reasons about metadata, which is how a hollow image passed. */ celiloVersion(ref: string): string | null; /** Names of leftover `celilo-e2e-*` containers. */ staleContainers(): string[]; /** Bytes docker reports as reclaimable image space. */ reclaimableImageBytes(): number; /** Per-test compose networks still up, with their attached containers. */ leakedStacks(): LeakedStack[]; } // ─── Dockerfile parsing ────────────────────────────────────────────── /** * External base images a Dockerfile pulls from, ignoring references to its own * earlier build stages (`FROM x AS fetch` … `COPY --from=fetch`). Pure so the * multi-stage and `--platform=` forms both stay covered by a unit test. */ export function parseBaseImages(dockerfile: string): string[] { const stages = new Set(); const bases: string[] = []; for (const raw of dockerfile.split('\n')) { const line = raw.trim(); const match = line.match(/^FROM\s+(.*)$/i); if (!match) continue; const tokens = match[1].split(/\s+/).filter((t) => !t.startsWith('--')); const ref = tokens[0]; if (!ref) continue; const asIdx = tokens.findIndex((t) => t.toLowerCase() === 'as'); if (asIdx >= 0 && tokens[asIdx + 1]) stages.add(tokens[asIdx + 1]); if (!stages.has(ref) && !bases.includes(ref)) bases.push(ref); } return bases; } /** Map `docker/Dockerfile.observer` → `celilo-e2e/observer` (docker-compose-generator's imageTag). */ function derivedImageTag(dockerfileName: string): string { return `celilo-e2e/${dockerfileName.replace(/^Dockerfile\./, '')}`; } export interface DockerfileBases { /** The `celilo-e2e/` tag this Dockerfile produces. */ derived: string; bases: string[]; } /** Read every `docker/Dockerfile.*` and pair its produced tag with its base images. */ export function readDockerfileBases(pkgDir: string): DockerfileBases[] { const dir = join(pkgDir, 'docker'); if (!existsSync(dir)) return []; return readdirSync(dir) .filter((f) => f.startsWith('Dockerfile.')) .sort() .map((f) => ({ derived: derivedImageTag(f), bases: parseBaseImages(readFileSync(join(dir, f), 'utf-8')), })); } // ─── Checks ────────────────────────────────────────────────────────── /** * The run-lock, including the two states `status` used to render identically to * a healthy one: a stale-heartbeat holder (suspect) and this session's own kept * stack (friction, not contention — the next run clears it). */ export function checkRunLock(lock: LockStatus): DoctorCheck { if (lock.free || !lock.holder) { return { name: 'run-lock', status: 'ok', detail: 'free' }; } const h = lock.holder; const age = formatAge(lock.heartbeatAgeMs ?? 0); if (lock.suspect) { return { name: 'run-lock', status: 'fail', detail: `SUSPECT — ${h.session} running ${h.test} (pid ${h.pid}) has not beaten its heartbeat for ${age}; the process is alive but is not progressing`, remedy: `Inspect it (\`docker ps\`, \`kill -0 ${h.pid}\`); if it is wedged, kill ${h.pid} then \`cele2e down\``, }; } if (lock.ownKept) { return { name: 'run-lock', status: 'warn', detail: `held by THIS session's own kept stack (${h.test}) — the next run auto-releases it`, remedy: 'cele2e release (or `cele2e down` to also tear the stack down)', }; } if (h.state === 'kept') { return { name: 'run-lock', status: 'fail', detail: `held by another session's kept stack: ${h.session} (${h.test})`, remedy: 'Wait for that session, or `cele2e release` if you know it is abandoned', }; } return { name: 'run-lock', status: 'fail', detail: `busy — ${h.session} running ${h.test} (pid ${h.pid}), heartbeat ${age} ago`, remedy: 'Poll with `cele2e status --json` (exit 0 = free, 3 = busy) and wait', }; } /** The baked management images exist AND still run /startup.sh. */ export function checkManagementImage(probe: DoctorProbe): DoctorCheck { const missing = BAKED_MANAGEMENT_IMAGES.filter((ref) => !probe.imageExists(ref)); if (missing.length > 0) { return { name: 'management-image', status: 'fail', detail: `baked image missing: ${missing.join(', ')} — the test compose references these by tag and cannot build them`, remedy: 'cele2e build-infra', }; } const unbaked = BAKED_MANAGEMENT_IMAGES.filter( (ref) => !(probe.imageCmd(ref) ?? []).includes(EXPECTED_MANAGEMENT_CMD), ); if (unbaked.length > 0) { return { name: 'management-image', status: 'fail', detail: `${unbaked.join(', ')} does not run ${EXPECTED_MANAGEMENT_CMD} — the bake commit lost its Cmd, so the ssh-keys volume will stay empty and every \`machine add\` will fail with "Cannot connect to root@ with provided SSH key"`, remedy: 'cele2e build-infra', }; } // The contract every stage depends on: a celilo binary that actually runs. // A failed bake (celilo#1365, the install.sh fetch timing out) leaves the // image present, correctly Cmd'd and completely hollow, and the suite then // dies at stage 1 with `celilo --version exited 127` — which reads as a test // defect rather than the bake failure it is. const version = probe.celiloVersion(MANAGEMENT_LATEST); if (!version) { return { name: 'management-image', status: 'fail', detail: `${MANAGEMENT_LATEST} is HOLLOW — it starts, but \`celilo\` is not on PATH inside it, so every stage dies with "celilo --version exited 127: command not found". The bake failed and left the image behind`, remedy: 'cele2e build-infra', }; } return { name: 'management-image', status: 'ok', detail: `${BAKED_MANAGEMENT_IMAGES.length} baked images present, both running ${EXPECTED_MANAGEMENT_CMD}; celilo ${version} inside ${MANAGEMENT_LATEST}`, }; } /** * Base images the next build genuinely has to fetch. * * A base image absent from the local store is only a problem when the image * built FROM it is also absent — with the derived tag present, buildkit serves * the build from cache and never resolves the reference. Requiring every base * unconditionally would refuse environments that work today, so the check is * the conjunction: no derived image AND no base image means the build must go * to a registry, which is where `failed to solve: ubuntu:22.04: net/http: TLS * handshake timeout` comes from — 16 images deep, blamed on the network. */ export function checkBaseImages(probe: DoctorProbe, dockerfiles: DockerfileBases[]): DoctorCheck { const needPull = new Set(); for (const { derived, bases } of dockerfiles) { if (probe.imageExists(derived)) continue; for (const base of bases) { if (!probe.imageExists(base)) needPull.add(base); } } if (needPull.size === 0) { return { name: 'base-images', status: 'ok', detail: 'every image the build needs is local' }; } const refs = [...needPull].sort(); return { name: 'base-images', status: 'fail', detail: `${refs.length} base image(s) absent locally and needed by an image that must be rebuilt: ${refs.join(', ')}`, remedy: refs.map((r) => `docker pull ${r}`).join('\n '), }; } /** * Epoch-ms a per-test stack started, parsed from its network's project name — * the attribution `docker network ls` carries for free. Null when the name is * not a per-test project (the shared and interactive projects hold no digits). */ export function stackStartedAt(network: string): number | null { const ms = network.match(/^celilo-e2e-(\d{13})_/)?.[1]; return ms ? Number(ms) : null; } /** * Networks from a crashed run that hold the sim's subnets. * * An active run holds the machine-global run-lock, so outside one there is no * legitimate holder of a `celilo-e2e-_*` network — it is always teardown * debris. Left in place it blocks EVERY later run with * `invalid pool request: Pool overlaps` at compose up (ce-ywix), a failure that * arrived only after minutes of image builds. The start-of-run sweep removes * these networks and their e2e-attached containers, so a cleanable leak is a * warning; a network blocked by a NON-e2e container is a failure, because the * sweep deliberately never touches foreign containers and the run cannot * succeed. */ export function checkLeakedStacks( probe: DoctorProbe, lock: LockStatus, callerHoldsLock = false, ): DoctorCheck { const stacks = probe.leakedStacks(); if (stacks.length === 0) { return { name: 'leaked-stacks', status: 'ok', detail: 'no leaked per-test stack networks' }; } // A live holder owns whatever per-test stack exists right now. `run` passes // callerHoldsLock instead: it already owns the lock, so at preflight time // every stack it can see is by definition somebody's debris, never its own. if (!callerHoldsLock && !lock.free && !lock.ownKept) { return { name: 'leaked-stacks', status: 'ok', detail: `${stacks.length} per-test network(s) belonging to the current lock holder`, }; } const describe = (s: LeakedStack): string => { const startedAt = stackStartedAt(s.network); const age = startedAt ? ` (suite started ${formatAge(Date.now() - startedAt)} ago)` : ''; const attached = s.containers.length > 0 ? `, attached: ${s.containers.join(', ')}` : ''; return `${s.network}${age}${attached}`; }; const foreign = stacks.flatMap((s) => s.containers.filter((c) => !c.startsWith(CONTAINER_PREFIX)), ); if (foreign.length > 0) { const blocked = stacks.filter((s) => s.containers.some((c) => !c.startsWith(CONTAINER_PREFIX))); return { name: 'leaked-stacks', status: 'fail', detail: `leaked per-test network(s) with a NON-e2e container attached — the cleanup sweep never touches foreign containers, so this stack cannot self-heal: ${blocked.map(describe).join('; ')}`, remedy: blocked .flatMap((s) => s.containers .filter((c) => !c.startsWith(CONTAINER_PREFIX)) .map((c) => `docker network disconnect -f ${s.network} ${c}`), ) .concat(blocked.map((s) => `docker network rm ${s.network}`)) .join('\n '), }; } return { name: 'leaked-stacks', status: 'warn', detail: `leaked per-test stack(s) holding the sim's subnets — always teardown debris, a later run's start-of-run cleanup removes them: ${stacks.slice(0, 3).map(describe).join('; ')}${stacks.length > 3 ? '; …' : ''}`, remedy: [ ...stacks.flatMap((s) => s.containers.length > 0 ? [`docker rm -f ${s.containers.join(' ')}`] : [], ), ...stacks.map((s) => `docker network rm ${s.network}`), ].join('\n '), }; } /** Leftover containers from a crashed run, which start-of-run cleanup will wipe. */ export function checkStaleContainers(probe: DoctorProbe, lock: LockStatus): DoctorCheck { const names = probe.staleContainers(); if (names.length === 0) { return { name: 'stale-containers', status: 'ok', detail: 'no leftover celilo-e2e-* containers', }; } // With a live holder these are simply that run's containers, not debris. if (!lock.free && !lock.ownKept) { return { name: 'stale-containers', status: 'ok', detail: `${names.length} celilo-e2e-* container(s) belonging to the current lock holder`, }; } return { name: 'stale-containers', status: 'warn', detail: `${names.length} celilo-e2e-* container(s) with no live run: ${names.slice(0, 4).join(', ')}${names.length > 4 ? ', …' : ''}`, remedy: 'cele2e down (start-of-run cleanup also removes these)', }; } /** * The Docker host's virtual machine, on the two settings that dominate run * time. Never a failure: a run on a badly-shaped VM is slow, not doomed, and * refusing to run would be a worse trade than saying so. Silent on Linux and * anywhere without a colima VM, where there is nothing to shape. * * See `host-vm.ts` for the measurements behind each rule. */ export function checkHostVm(facts: HostVmFacts | null, host: HostFacts): DoctorCheck { if (!facts) { return { name: 'docker-host', status: 'ok', detail: 'no colima VM — docker runs natively, no host-VM policy applies', }; } const budget = recommendedBudget(host, facts.vmType); const { problems, needsRecreate } = evaluateHostVm(facts, host, budget); if (problems.length === 0) { return { name: 'docker-host', status: 'ok', detail: `colima/${facts.profile}: ${facts.cpus} CPU, ${facts.memoryGiB} GiB, ${facts.mountType} mounts`, }; } return { name: 'docker-host', status: 'warn', detail: problems.join(' '), remedy: needsRecreate ? 'cele2e host reset (colima discards a mount-type change on an existing VM, so this DESTROYS the VM and its image cache and rebuilds — budget one `cele2e build-infra`)' : 'cele2e host up (restarts the VM with the recommended budget; images survive)', }; } /** * Was the baked management image built from the source in this working tree? * * The image carries a real installed `celilo`, and that CLI is what every test * exercises. A stale one produces a run that passes or fails for reasons the * working tree cannot explain, with nothing in the output to say so. Comparing * a fingerprint stamped at bake time against one computed now turns that into a * sentence. * * A warning, not a failure — deliberately testing an older image is a real * thing to do, and this check's job is to make sure it is deliberate. */ export function checkImageFreshness(probe: DoctorProbe, repoRoot: string | null): DoctorCheck { const expected = computeSourceFingerprint(repoRoot ?? undefined); if (!expected) { return { name: 'image-freshness', status: 'ok', detail: 'no celilo checkout to compare the baked image against', }; } const stamped = probe.imageLabel(MANAGEMENT_LATEST, SOURCE_LABEL); if (stamped === null) { return { name: 'image-freshness', status: 'warn', detail: `${MANAGEMENT_LATEST} carries no source stamp, so it predates this check and its age cannot be established`, remedy: 'cele2e build-infra', }; } if (stamped.startsWith(PUBLISHED_FINGERPRINT_PREFIX)) { return { name: 'image-freshness', status: 'ok', detail: `${MANAGEMENT_LATEST} was baked from ${stamped} (real npm), not from this tree — nothing to be stale against`, }; } if (stamped !== expected) { return { name: 'image-freshness', status: 'warn', detail: `${MANAGEMENT_LATEST} was baked from source ${stamped}; this tree is ${expected}. The celilo CLI under test is NOT the code in this checkout`, remedy: 'cele2e build-infra', }; } return { name: 'image-freshness', status: 'ok', detail: `${MANAGEMENT_LATEST} was baked from this tree (${stamped})`, }; } /** * Reclaimable image space. A warning, never a failure — the point is to steer * the reflex: under disk pressure people reach for `docker image prune`, which * deletes the base images the next build needs. Pruning e2e containers by name * frees space without costing a 27-image rebuild. */ export function checkDiskPressure(probe: DoctorProbe): DoctorCheck { const bytes = probe.reclaimableImageBytes(); const gib = (bytes / 1024 ** 3).toFixed(1); if (bytes < RECLAIMABLE_WARN_BYTES) { return { name: 'disk', status: 'ok', detail: `${gib} GiB reclaimable image space` }; } return { name: 'disk', status: 'warn', detail: `${gib} GiB reclaimable image space — most of it is superseded management images, which \`cele2e build-infra\` now removes as it goes`, remedy: 'docker image prune -f (UNTAGGED images only — safe. NEVER `-a` or `docker system prune`, which delete the TAGGED base images the next build needs and cost a full 27-image rebuild)', }; } /** * Turn buildkit's `failed to solve: : … TLS handshake timeout` into the * one sentence that identifies the cause. * * That message names the network because buildkit was, technically, on the * network — but it only went there because the base image was not in the local * store, which is what `docker image prune` (the reflex under disk pressure) * removes. Reading it as a connectivity problem cost a full 27-image rebuild. * checkBaseImages preflights for this, so reaching here means the image went * missing mid-run; either way, say what to pull. * * Returns '' when the failure names a RUN step rather than an image reference — * a genuine build error, which needs no translation. */ export function explainBuildFailure(stderr: string): string { const ref = stderr.match(/failed to solve:\s*([^\s:"]+(?::[^\s:"]+)?)/)?.[1]; if (!ref || ref === 'process') return ''; return [ '', '', `This names "${ref}" — buildkit had to fetch it, which means it is not in the local image store.`, `A \`docker image prune\` removes exactly these. Fix: docker pull ${ref}`, 'Then re-run. `cele2e doctor` checks for this before a run starts.', ].join('\n'); } // ─── Real probe ────────────────────────────────────────────────────── function docker(args: string[]): string { return execFileSync('docker', args, { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 15_000, }).trim(); } export function createDockerProbe(): DoctorProbe { // One `docker images` listing answers every existence question; per-image // inspect calls would be ~40 process spawns on a preflight that must be fast. let tags: Set | null = null; const knownTags = (): Set => { if (tags) return tags; try { tags = new Set( docker(['images', '--format', '{{.Repository}}:{{.Tag}}']) .split('\n') .map((l) => l.trim()) .filter(Boolean), ); } catch { tags = new Set(); } return tags; }; return { imageExists(ref) { const withTag = ref.includes(':') ? ref : `${ref}:latest`; return knownTags().has(withTag); }, imageCmd(ref) { try { return JSON.parse(docker(['image', 'inspect', ref, '--format', '{{json .Config.Cmd}}'])); } catch { return null; } }, imageLabel(ref, label) { try { const out = docker([ 'image', 'inspect', ref, '--format', `{{index .Config.Labels "${label}"}}`, ]); // Go's text/template renders a missing map key as this literal. return out === '' || out === '' ? null : out; } catch { return null; } }, celiloVersion(ref) { try { const out = docker([ 'run', '--rm', '--entrypoint', 'sh', ref, '-c', 'command -v celilo >/dev/null 2>&1 && celilo --version', ]); return out === '' ? null : out.split('\n').pop()?.trim() || null; } catch { return null; } }, staleContainers() { try { return docker(['ps', '-a', '--filter', 'name=celilo-e2e', '--format', '{{.Names}}']) .split('\n') .map((l) => l.trim()) .filter(Boolean); } catch { return []; } }, leakedStacks() { // Per-test compose networks only: the project carries the epoch-ms the // suite started, so a digits-then-underscore name cannot match the // shared (`celilo-e2e-shared_*`) or interactive (`celilo-e2e-interactive`) // stacks, both of which can be legitimately up outside a run. const perTest = (n: string): boolean => /^celilo-e2e-\d{13}_/.test(n); try { const nets = docker(['network', 'ls', '--format', '{{.Name}}']) .split('\n') .map((l) => l.trim()) .filter(perTest); const stacks: LeakedStack[] = []; for (const net of nets) { try { const containers = JSON.parse( docker(['network', 'inspect', net, '--format', '{{json .Containers}}']), ) as Record; stacks.push({ network: net, containers: Object.values(containers) .map((c) => c.Name ?? '') .filter(Boolean), }); } catch { stacks.push({ network: net, containers: [] }); } } return stacks; } catch { return []; } }, reclaimableImageBytes() { try { for (const line of docker(['system', 'df', '--format', '{{json .}}']).split('\n')) { const row = JSON.parse(line) as { Type?: string; Reclaimable?: string }; if (row.Type !== 'Images') continue; // Reclaimable looks like "12.3GB (48%)". const m = row.Reclaimable?.match(/^([\d.]+)\s*([KMGT]?)B/i); if (!m) return 0; const scale = { '': 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }[ m[2].toUpperCase() ]; return Number.parseFloat(m[1]) * (scale ?? 1); } } catch {} return 0; }, }; } // ─── Report ────────────────────────────────────────────────────────── export interface DiagnoseOptions { pkgDir: string; probe?: DoctorProbe; lock?: LockStatus; /** Skip the run-lock check — `run` acquires the lock itself and reports contention its own way. */ skipLock?: boolean; /** * The celilo checkout to fingerprint against. Defaults to a walk-up from * `pkgDir`, so a caller cannot silently disable the freshness check by * forgetting to pass it — the failure mode this repo calls "a check that * cannot reach the thing it is checking". */ repoRoot?: string | null; /** The running VM and the host it sits on. Injected so both are testable. */ hostVm?: HostVmFacts | null; host?: HostFacts; } export function diagnose(options: DiagnoseOptions): DoctorReport { const probe = options.probe ?? createDockerProbe(); const lock = options.lock ?? lockStatus(); const checks: DoctorCheck[] = []; if (!options.skipLock) checks.push(checkRunLock(lock)); checks.push(checkManagementImage(probe)); checks.push(checkBaseImages(probe, readDockerfileBases(options.pkgDir))); checks.push(checkStaleContainers(probe, lock)); checks.push( checkImageFreshness( probe, options.repoRoot !== undefined ? options.repoRoot : findMonorepoRoot(options.pkgDir), ), ); checks.push( checkHostVm( options.hostVm !== undefined ? options.hostVm : readHostVmFacts(), options.host ?? readHostFacts(), ), ); checks.push(checkLeakedStacks(probe, lock, options.skipLock === true)); checks.push(checkDiskPressure(probe)); return { checks, ok: !checks.some((c) => c.status === 'fail') }; } const ICONS: Record = { ok: '✓', warn: '!', fail: '✗' }; /** Render a report as the lines `doctor` prints and `run` prints on refusal. */ export function formatReport(report: DoctorReport): string[] { const lines: string[] = []; for (const c of report.checks) { lines.push(` ${ICONS[c.status]} ${c.name.padEnd(18)} ${c.detail}`); if (c.remedy) lines.push(` fix: ${c.remedy}`); } return lines; } /** The holder line `status` prints, with heartbeat age always visible. */ export function formatHolderLine(h: LockHolder): string { const flags = [ isSuspect(h) ? 'SUSPECT' : null, h.state === 'kept' && isSameSession(h) ? 'yours' : null, ].filter(Boolean); const suffix = flags.length ? ` [${flags.join(', ')}]` : ''; return `${h.session} — ${h.state} ${h.test} (pid ${h.pid}), heartbeat ${formatAge(heartbeatAgeMs(h))} ago${suffix}`; }