// `qa-status [path]`: report where a qa-run stands from its on-disk state: // live (launching, queued, running), dead (non-terminal status document with // a dead PID), or terminal (result document written; the state is the run's // verdict). Classification is fail-closed: liveness is proven by the PID, // never assumed from the status document, and the result document is // authoritative over the status document once it exists. `--wait` reconnects // to a detached run and exits with its verdict; `--queue` shows the // machine-wide admission queue instead of any one run. import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import type { Command } from "commander"; import type { EmitContext, HarneryProgramContext } from "../commander.ts"; import { latestManagedQaRun, managedQaReviewGuidance, resolveQaRepoRoot, } from "../core/qa-artifacts.ts"; import { type AdmissionEntry, admissionBaseDir, admissionStatus, pidAlive as defaultPidAlive, listAdmissionResources, } from "../lib/admission.ts"; import { QA_RUN_LATEST_FILENAME, QA_RUN_RESULT_FILENAME, QA_RUN_STATUS_FILENAME, } from "../lib/browser/qa-run.ts"; import { QA_RUN_STAGES, type QaRunStage, type QaRunVerdict, } from "../lib/browser/qa-run-contracts.ts"; import { QA_ADMISSION_RESOURCE } from "./qa-run.ts"; /** A running state whose heartbeat is older than this earns a staleness * warning, but stays "running" while the PID is alive. */ export const QA_STATUS_HEARTBEAT_STALE_MS = 120_000; /** Log file `qa-run --detach` leaves in the run directory. */ const RUNNER_LOG_FILENAME = "runner.log"; const POLL_MS = 2_000; const DEFAULT_WAIT_TIMEOUT_MINUTES = 60; const STATUS_DOC_STATES = new Set(["launching", "queued", "running", "completed"]); function describeError(err: unknown): string { return err instanceof Error ? err.message : String(err); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } type JsonRead = { ok: true; value: unknown } | { ok: false; error: string }; function readJsonFile(path: string): JsonRead { let raw: string; try { raw = readFileSync(path, "utf8"); } catch (err: unknown) { return { ok: false, error: `cannot read ${path}: ${describeError(err)}` }; } try { return { ok: true, value: JSON.parse(raw) }; } catch (err: unknown) { return { ok: false, error: `${path} is not valid JSON: ${describeError(err)}` }; } } // --------------------------------------------------------------------------- // Path resolution // --------------------------------------------------------------------------- export type RunDirResolution = { ok: true; runDir: string } | { ok: false; error: string }; function hasRunFiles(dir: string): boolean { return ( existsSync(join(dir, QA_RUN_STATUS_FILENAME)) || existsSync(join(dir, QA_RUN_RESULT_FILENAME)) ); } /** Best-effort started_at for one run directory: the status document's * `started_at`, falling back to the result document's `run.started_at`. */ function startedAtOf(dir: string): string | undefined { const statusPath = join(dir, QA_RUN_STATUS_FILENAME); if (existsSync(statusPath)) { const read = readJsonFile(statusPath); if (read.ok && isRecord(read.value) && typeof read.value.started_at === "string") { return read.value.started_at; } } const resultPath = join(dir, QA_RUN_RESULT_FILENAME); if (existsSync(resultPath)) { const read = readJsonFile(resultPath); if (read.ok && isRecord(read.value) && isRecord(read.value.run)) { const startedAt = read.value.run.started_at; if (typeof startedAt === "string") return startedAt; } } return undefined; } /** * Resolve `[path]` to a run directory. A file is a result document (its * directory is the run directory); a directory carrying a status or result * document is the run directory itself; any other directory is treated as a * run PARENT: its `run-*` children are scanned and the one with the newest * started_at wins, falling back to the `latest.json` pointer when no child * carries a status or result. Without a path, the newest managed QA artifact * under the repository's `.harnery/artifacts/` store is used. */ export function resolveRunDir(inputPath?: string, repoRoot?: string): RunDirResolution { if (inputPath === undefined) { const latest = latestManagedQaRun(repoRoot ?? resolveQaRepoRoot()); if (!latest) { return { ok: false, error: "no managed qa-run or qa-record output exists under .harnery/artifacts", }; } return { ok: true, runDir: latest }; } const abs = resolve(inputPath); let stats: ReturnType; try { stats = statSync(abs); } catch (err: unknown) { return { ok: false, error: `cannot access ${abs}: ${describeError(err)}` }; } if (stats.isFile()) return { ok: true, runDir: dirname(abs) }; if (!stats.isDirectory()) { return { ok: false, error: `${abs} is neither a file nor a directory` }; } if (hasRunFiles(abs)) return { ok: true, runDir: abs }; // Run parent: newest run-*/ child by started_at. let childNames: string[] = []; try { childNames = readdirSync(abs, { withFileTypes: true }) .filter((entry) => entry.isDirectory() && entry.name.startsWith("run-")) .map((entry) => entry.name) .sort(); } catch { childNames = []; } let best: { dir: string; startedAtMs: number } | undefined; for (const name of childNames) { const child = join(abs, name); if (!hasRunFiles(child)) continue; const startedAt = startedAtOf(child); const startedAtMs = startedAt !== undefined ? Date.parse(startedAt) : Number.NaN; if (Number.isNaN(startedAtMs)) continue; if (!best || startedAtMs > best.startedAtMs) best = { dir: child, startedAtMs }; } if (best) return { ok: true, runDir: best.dir }; const pointerPath = join(abs, QA_RUN_LATEST_FILENAME); if (existsSync(pointerPath)) { const pointer = readJsonFile(pointerPath); if (!pointer.ok) return pointer; if (isRecord(pointer.value)) { const record = pointer.value; let rel: string | undefined; if (typeof record.dir === "string" && record.dir.length > 0) rel = record.dir; else if (typeof record.result === "string" && record.result.length > 0) { rel = dirname(record.result); } if (rel !== undefined) return { ok: true, runDir: resolve(abs, rel) }; } return { ok: false, error: `${pointerPath} carries no run directory to follow` }; } return { ok: false, error: `${abs} carries no ${QA_RUN_STATUS_FILENAME} or ${QA_RUN_RESULT_FILENAME}, no run-*/ ` + `child with either, and no ${QA_RUN_LATEST_FILENAME} pointer`, }; } // --------------------------------------------------------------------------- // Classification // --------------------------------------------------------------------------- export type QaStatusState = QaRunVerdict | "launching" | "queued" | "running" | "dead"; export interface QaStatusReport { state: QaStatusState; /** The result document exists; state is the run's final verdict. */ terminal: boolean; /** 0 completed-passed · 2 completed-failed · 4 completed-incomplete or * dead · 5 still in progress. */ exit: 0 | 2 | 4 | 5; run_id: string | null; pid: number | null; stage: QaRunStage | null; started_at: string | null; updated_at: string | null; completed_at: string | null; /** now minus the status document's updated_at; null without a heartbeat. */ heartbeat_age_ms: number | null; verdict?: QaRunVerdict; result_path?: string; log_path?: string; queue?: { resource: string; waiting_since: string }; wall_time_ms?: Record; warnings: string[]; } export interface QaStatusDeps { /** PID liveness probe (injectable for tests). Default: process.kill(pid, 0). */ pidAlive?: (pid: number) => boolean; /** Evaluation instant in epoch milliseconds. Default: Date.now(). */ now?: () => number; } export type QaStatusClassification = | { ok: true; report: QaStatusReport } | { ok: false; error: string }; function exitForVerdict(verdict: QaRunVerdict): 0 | 2 | 4 { if (verdict === "passed") return 0; if (verdict === "failed") return 2; return 4; } /** * Classify one run directory, fail-closed. The result document is terminal * and authoritative when it exists; otherwise the status document plus a PID * liveness probe decides between a live state and "dead". Neither file is a * usage error, not a state. */ export function classifyRun(runDir: string, deps: QaStatusDeps = {}): QaStatusClassification { const isAlive = deps.pidAlive ?? defaultPidAlive; const nowMs = deps.now ? deps.now() : Date.now(); const abs = resolve(runDir); const resultPath = join(abs, QA_RUN_RESULT_FILENAME); const statusPath = join(abs, QA_RUN_STATUS_FILENAME); const logPath = join(abs, RUNNER_LOG_FILENAME); const logExists = existsSync(logPath); const resultExists = existsSync(resultPath); // The status document supplies pid/stage/heartbeat even for terminal runs, // so read it best-effort first; a broken status only matters when it is // the sole evidence. let status: Record | undefined; if (existsSync(statusPath)) { const read = readJsonFile(statusPath); if (read.ok && isRecord(read.value)) status = read.value; else if (!resultExists) { return { ok: false, error: read.ok ? `${statusPath} is not a status object` : read.error }; } } let heartbeatAgeMs: number | null = null; if (status && typeof status.updated_at === "string") { const updatedMs = Date.parse(status.updated_at); if (!Number.isNaN(updatedMs)) heartbeatAgeMs = Math.max(0, nowMs - updatedMs); } if (resultExists) { const read = readJsonFile(resultPath); if (!read.ok) return read; if (!isRecord(read.value)) { return { ok: false, error: `${resultPath} is not a result object` }; } const result = read.value; const verdict = result.verdict; if (verdict !== "passed" && verdict !== "failed" && verdict !== "incomplete") { return { ok: false, error: `${resultPath} carries no recognizable verdict` }; } const run = isRecord(result.run) ? result.run : {}; const statusRunId = typeof status?.run_id === "string" ? status.run_id : null; return { ok: true, report: { state: verdict, terminal: true, exit: exitForVerdict(verdict), run_id: typeof run.run_id === "string" ? run.run_id : statusRunId, pid: typeof status?.pid === "number" ? status.pid : null, stage: null, started_at: typeof run.started_at === "string" ? run.started_at : null, updated_at: typeof status?.updated_at === "string" ? status.updated_at : null, completed_at: typeof run.completed_at === "string" ? run.completed_at : null, heartbeat_age_ms: heartbeatAgeMs, verdict, result_path: resultPath, ...(logExists ? { log_path: logPath } : {}), ...(isRecord(result.wall_time_ms) ? { wall_time_ms: result.wall_time_ms as Record } : {}), warnings: [], }, }; } if (!status) { return { ok: false, error: `${abs} carries neither ${QA_RUN_RESULT_FILENAME} nor ${QA_RUN_STATUS_FILENAME}: ` + "not a qa-run directory", }; } const docState = typeof status.state === "string" ? status.state : ""; if (!STATUS_DOC_STATES.has(docState)) { return { ok: false, error: `${statusPath} carries unrecognized state ${JSON.stringify(status.state)}`, }; } const pid = typeof status.pid === "number" ? status.pid : 0; const runId = typeof status.run_id === "string" ? status.run_id : null; const stage = typeof status.stage === "string" && (QA_RUN_STAGES as readonly string[]).includes(status.stage) ? (status.stage as QaRunStage) : null; const startedAt = typeof status.started_at === "string" ? status.started_at : null; const updatedAt = typeof status.updated_at === "string" ? status.updated_at : null; const alive = pid > 0 && isAlive(pid); if (!alive) { return { ok: true, report: { state: "dead", terminal: false, exit: 4, run_id: runId, pid: pid > 0 ? pid : null, stage, started_at: startedAt, updated_at: updatedAt, completed_at: null, heartbeat_age_ms: heartbeatAgeMs, log_path: logPath, warnings: [], }, }; } const warnings: string[] = []; let state: QaStatusState; if (docState === "completed") { // The status document claims completion but no result document exists. // The PID is alive, so the runner may still be writing; stay "running" // and say so rather than inventing a verdict. state = "running"; warnings.push( `${QA_RUN_STATUS_FILENAME} says completed but no ${QA_RUN_RESULT_FILENAME} exists yet; ` + "treating as running while the PID is alive", ); } else { state = docState as "launching" | "queued" | "running"; } if ( state === "running" && heartbeatAgeMs !== null && heartbeatAgeMs > QA_STATUS_HEARTBEAT_STALE_MS ) { warnings.push( `heartbeat stale: last update ${Math.round(heartbeatAgeMs / 1000)}s ago ` + `(threshold ${QA_STATUS_HEARTBEAT_STALE_MS / 1000}s), but pid ${pid} is alive`, ); } const queueValue = status.queue; const queue = isRecord(queueValue) && typeof queueValue.resource === "string" && typeof queueValue.waiting_since === "string" ? { resource: queueValue.resource, waiting_since: queueValue.waiting_since } : undefined; return { ok: true, report: { state, terminal: false, exit: 5, run_id: runId, pid, stage, started_at: startedAt, updated_at: updatedAt, completed_at: null, heartbeat_age_ms: heartbeatAgeMs, ...(queue ? { queue } : {}), ...(logExists ? { log_path: logPath } : {}), warnings, }, }; } // --------------------------------------------------------------------------- // Rendering // --------------------------------------------------------------------------- function formatAge(ms: number): string { const totalSeconds = Math.max(0, Math.round(ms / 1000)); if (totalSeconds < 60) return `${totalSeconds}s`; const minutes = Math.floor(totalSeconds / 60); if (minutes < 60) return `${minutes}m ${totalSeconds % 60}s`; const hours = Math.floor(minutes / 60); return `${hours}h ${minutes % 60}m`; } function renderReport(emit: EmitContext, report: QaStatusReport): void { const runId = report.run_id ?? "(no run id)"; if (report.terminal) { const completed = report.completed_at ? `, completed ${report.completed_at}` : ""; emit.log( `run ${runId}: verdict ${report.verdict}${completed} (${report.result_path})`, report.state === "passed" ? "info" : "warn", ); if (report.wall_time_ms) { const total = report.wall_time_ms.total; const queueMs = report.wall_time_ms.queue; const totalPart = total !== undefined ? `total ${total}ms` : "total unknown"; const queuePart = queueMs !== undefined ? `, queue wait ${queueMs}ms` : ""; emit.log(`wall time: ${totalPart}${queuePart}`, "info"); } } else if (report.state === "dead") { const bits = [ `pid ${report.pid ?? "unknown"} not running`, ...(report.stage !== null ? [`last stage ${report.stage}`] : []), ...(report.heartbeat_age_ms !== null ? [`last heartbeat ${formatAge(report.heartbeat_age_ms)} ago`] : []), ]; emit.log(`run ${runId}: dead, ${bits.join(", ")}`, "warn"); if (report.log_path !== undefined) emit.log(`log: ${report.log_path}`, "warn"); } else { const bits = [ ...(report.stage !== null ? [`stage ${report.stage}`] : []), `pid ${report.pid ?? "unknown"}`, ...(report.heartbeat_age_ms !== null ? [`heartbeat ${formatAge(report.heartbeat_age_ms)} ago`] : []), ]; emit.log(`run ${runId}: ${report.state}, ${bits.join(", ")}`, "info"); if (report.queue) { emit.log(`queued for ${report.queue.resource} since ${report.queue.waiting_since}`, "info"); } } for (const warning of report.warnings) emit.log(`warning: ${warning}`, "warn"); } function describeAdmissionEntry( entry: AdmissionEntry, nowMs: number, verb: "acquired" | "queued", ): string { const stamp = verb === "acquired" ? (entry.acquired_at ?? entry.created_at) : entry.created_at; const ageMs = nowMs - Date.parse(stamp); const age = Number.isNaN(ageMs) ? `${verb} at unknown time` : `${verb} ${formatAge(ageMs)} ago`; return `${entry.label || "(no label)"} (pid ${entry.pid}), ${age}`; } function runQueueAction(emit: EmitContext, json: boolean): void { const dir = admissionBaseDir(); const nowMs = Date.now(); const names = new Set(listAdmissionResources(dir)); names.add(QA_ADMISSION_RESOURCE); // browser-qa first, everything else alphabetical. const ordered = [...names].sort((a, b) => { if (a === QA_ADMISSION_RESOURCE) return -1; if (b === QA_ADMISSION_RESOURCE) return 1; return a.localeCompare(b); }); const resources = ordered.map((resource) => admissionStatus({ dir, resource })); if (json) { emit.data({ dir, resources }); return; } for (const entry of resources) { const holders = entry.holders.length; const waiters = entry.waiters.length; emit.log( `${entry.resource}: ${holders} holder${holders === 1 ? "" : "s"}, ${waiters} waiting`, "info", ); for (const holder of entry.holders) { emit.log(` holding: ${describeAdmissionEntry(holder, nowMs, "acquired")}`, "info"); } for (const waiter of entry.waiters) { emit.log(` waiting: ${describeAdmissionEntry(waiter, nowMs, "queued")}`, "info"); } } } // --------------------------------------------------------------------------- // Command // --------------------------------------------------------------------------- interface QaStatusOpts { json?: boolean; wait?: boolean; waitTimeout?: string; queue?: boolean; } function sleep(ms: number): Promise { return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); } export function registerQaStatusCommand( program: Command, emit: EmitContext, context?: HarneryProgramContext, ): void { program .command("qa-status [path]") .description( "Report where a qa-run stands from its on-disk state: live (launching, queued, " + "running), dead (non-terminal status with a dead PID), or terminal (result " + "document written; the state is the verdict). Fail-closed: liveness is proven " + "by the PID, never assumed from the status document.", ) .option("--json", "Print the status envelope as JSON.") .option("--wait", "Poll every 2s until the run reaches a terminal verdict or is dead.") .option( "--wait-timeout ", `Maximum --wait duration in minutes (default ${DEFAULT_WAIT_TIMEOUT_MINUTES}).`, ) .option( "--queue", "Ignore [path] and print the machine-wide admission queue instead: holders and " + "waiters per resource, dead entries pruned on the way.", ) .addHelpText( "after", `\n[path] may be a ${QA_RUN_RESULT_FILENAME} file, a run directory (containing ` + `${QA_RUN_STATUS_FILENAME} or ${QA_RUN_RESULT_FILENAME}), or a run parent directory ` + `(newest run-*/ child by started_at, falling back to the ${QA_RUN_LATEST_FILENAME} ` + `pointer). Without [path], the newest managed run under .harnery/artifacts is used.` + "\n\nExit codes: 0 completed passed · 1 usage or unreadable state · " + "2 completed failed · 4 completed incomplete or dead · " + "5 still in progress (or --wait timeout).", ) .action(async (path: string | undefined, opts: QaStatusOpts) => { if (opts.queue) { runQueueAction(emit, opts.json === true); return; } let waitTimeoutMinutes = DEFAULT_WAIT_TIMEOUT_MINUTES; if (opts.waitTimeout !== undefined) { const minutes = Number.parseInt(opts.waitTimeout, 10); if (!Number.isInteger(minutes) || minutes < 1) { emit.error({ code: "qa_status_invalid_wait_timeout", message: "--wait-timeout must be a positive integer number of minutes", }); process.exitCode = 1; return; } waitTimeoutMinutes = minutes; } const resolved = resolveRunDir(path, resolveQaRepoRoot(context)); if (!resolved.ok) { emit.error({ code: "qa_status_unresolvable_path", message: resolved.error }); process.exitCode = 1; return; } const runDir = resolved.runDir; const deadline = Date.now() + waitTimeoutMinutes * 60_000; let lastProgressLine = ""; while (true) { const outcome = classifyRun(runDir); if (!outcome.ok) { emit.error({ code: "qa_status_unreadable", message: outcome.error }); process.exitCode = 1; return; } const report = outcome.report; const settled = report.terminal || report.state === "dead"; if (settled || !opts.wait) { const afterReview = report.terminal ? managedQaReviewGuidance(resolveQaRepoRoot(context), dirname(runDir)) : undefined; if (opts.json) emit.data({ ...report, ...(afterReview ? { after_review: afterReview } : {}) }); renderReport(emit, report); if (afterReview) emit.log(afterReview, "info"); if (!settled && !opts.wait) { emit.log("still in progress; rerun with --wait to block until the run settles", "info"); } if (report.exit !== 0) process.exitCode = report.exit; return; } const progressLine = `run ${report.run_id ?? "(no run id)"}: ${report.state}` + `${report.stage !== null ? `, stage ${report.stage}` : ""}`; if (progressLine !== lastProgressLine) { lastProgressLine = progressLine; emit.log(progressLine, "info"); } if (Date.now() >= deadline) { if (opts.json) emit.data(report); emit.log( `wait timed out after ${waitTimeoutMinutes}m; run is still ${report.state}`, "warn", ); process.exitCode = 5; return; } await sleep(POLL_MS); } }); }