/** * vigiles — run harness-test / eval script files via the CLI. * * `vigiles test` and `vigiles eval` discover `*.harness.*` / `*.eval.*` * scripts and run each as a child `node` process, so the two-tier * harness-testing API (`src/harness-test.ts`, `src/eval.ts`) works as a CI * command, not just `node x.mjs`. Scripts may be authored in **JavaScript** * (`.mjs` / `.cjs` / `.js`) **or TypeScript** (`.ts` / `.mts` / `.cts`) — a TS * script is run through `tsx` when installed, else Node's built-in type * stripping (Node >= 22.6). The scripts import from the built `dist/`, so they * also run standalone — the CLI just discovers, runs, and aggregates exit codes. */ import { type TestTier } from "../../source-kinds.js"; import { type SurfaceProbe } from "../../check-count.js"; /** * The outcome of running one script. * * `"vacuous"` — the script exited 0 and reported that it made ZERO checks. It * neither passed nor failed: nothing was verified. See {@link statusFor}. */ export type ScriptStatus = "pass" | "skip" | "fail" | "vacuous"; export interface ScriptRunResult { readonly file: string; readonly code: number; readonly status: ScriptStatus; /** * How many checks the script reported making, or `undefined` when it reported * nothing at all — a script that never imports `vigiles` has no way to * report, and that silence is NOT a claim about it. `0` is a claim: the script * loaded the library and used none of it. */ readonly checks?: number; /** * The surfaces this script was seen to exercise — derived by the tiers from the * command they ran and the transcripts they got back, never declared by the * author (see `coverage-probe.ts`). Feeds `.vigiles/coverage.json`, which lets * coverage answer "tested?" by EXECUTION instead of by file name. * * Absent for a script that reported nothing, and empty for one that reported a * count but exercised no identifiable surface — a unit test of a pure helper, * say. Neither is a finding. */ readonly surfaces?: readonly SurfaceProbe[]; } /** * Exit code a harness/eval script uses to report itself SKIPPED (e.g. the * deterministic tier when `claude` isn't installed) — the autotools convention. * The runner surfaces it as a loud `⊘ SKIPPED` instead of a silent `✓`, and a * skip never fails the run. Scripts call `skip()` (vigiles) to emit it. */ export declare const SKIP_EXIT_CODE = 77; /** * What the runner's load hook saw of one script (`harness-resolve-hooks.mts`). * * `marked` — the hook saw the script's module as an ES module and planted a * marker import at the top of it. `linked` — that marker evaluated, which ESM * only does once the WHOLE import graph was found, parsed and linked. */ export interface LoadEvidence { readonly marked: boolean; readonly linked: boolean; } /** * Classify one script's run from its exit code, its reported check count and * what the load hook saw. * * 🔴 THE FOURTH STATE, AND WHY. Exit codes answer "did it fail?", never "did it * do anything?". Measured 2026-08-08: a file whose whole body is * `export default { "never runs": () => assert.equal(1, 2) }` imports fine, * exits 0, and printed `✓ … 1 passed` — a false assertion, never called, * reported as a pass. A consumer repo hit exactly that and now hand-copies a * warning into every new harness header, because the runner could not enforce * it: eight harnesses resting on a comment. * * So a run that ends clean having recorded ZERO checks is `"vacuous"` — its own * visible state, not folded into `passed`, the same way a skip is not. * * NOT A FAILURE, deliberately. Harnesses in the wild predate the counter, and a * tool that turned CI red on the release that taught it a new word would be * punishing people for upgrading. It is loud and it is not fatal. * * AND SILENCE IS NOT ZERO. `checks === undefined` means the script never * reported — it may not import `vigiles` at all — so it stays a plain * `pass`, exactly as before. Only a script that loaded the library and used * none of it says zero. This is the `undefined`-vs-`[]` distinction * `assertNoWrite` already draws: "nobody looked" must not read as "nothing * happened". * * 🔴 DID IT LOAD? A non-zero exit from a script whose module was marked and * never linked is did-not-load: `"skip"` with the loader's exit code, which * keeps its coverage (`fail` RETRACTS it, see `runsFromResults`). A module that * never linked executed no assertion — the same "did not run" as a missing * `claude`, arriving through a different door. MEASURED, twice on 2026-08-20: a * container restore left an old `node_modules`, every harness in a consumer repo * died on `Named export 'recordCheck' not found`, and its ledger fell from 48 * records to 34 and from 47 to 33 for surfaces nothing had touched. The run is * still red: `cli-main.ts` fails on any skip the author did not declare. * * This used to be decided by grepping the child's OUTPUT for loader phrases, * and that was wrong in the dangerous direction (#243): a harness that printed * a hook transcript containing "Cannot find module", then failed an assertion, * was read as did-not-load and kept its coverage. Output is no longer an input. * Anything without a marker (CommonJS, an entry the hook never saw) can never * claim did-not-load — it is a `fail`, the side that retracts rather than hides. * A syntax error in the script itself also never links, so it too is * did-not-load; that is the owner's call, not an accident. */ export declare function statusFor(code: number, checks: number | undefined, load?: LoadEvidence): ScriptStatus; /** * Did this script fail to LOAD — as opposed to declaring a skip (exit 77)? * The one predicate behind both the CLI's "never ran" failure and `--min`. * * Deliberately NOT a fifth `ScriptStatus`: coverage retraction reads the status * as a bare string (`coverage-artifact.ts`), so a new member would start * retracting silently with no type error. */ export declare function loadFailed(r: ScriptRunResult): boolean; /** * Filename extensions accepted for harness/eval scripts (JS and TS). The table * lives in `source-kinds.ts`; this re-export keeps the runner's name for it. */ export declare const SCRIPT_EXTS: readonly ["mjs", "cjs", "js", "mts", "cts", "ts"]; /** The glob that discovers one tier's scripts, derived from the same table as everything else that names a test. */ export declare function scriptGlob(kind: TestTier): string; export { detectNodeCaps, canRunTypeScript, type NodeCaps, } from "../../ts-runner-caps.js"; import type { NodeCaps } from "../../ts-runner-caps.js"; import { type GlobIgnore } from "../../core/glob-ignore.js"; /** * The `node` argv (after the binary) to run a single script. Plain JS runs * directly; a TypeScript script picks `tsx` when available, else Node's native * type stripping. Throws a clear, actionable error when neither is available. * Pure — exported for testing. * * `entry` interposes a program that takes the script as its ARGUMENT instead of * running the script as the program. `vigiles eval` passes one: an eval file * describes its eval rather than running it (see `src/eval-define.ts`), so * something has to import the description and execute what it declares. Harness * scripts pass nothing and are launched exactly as before. * * 🔴 The disjunction below is `canRunTypeScript` — keep them together. When they * drifted, the tool recommended a `.ts` file and then refused to run it. */ export declare function interpreterArgs(file: string, caps: NodeCaps, entry?: string): string[]; /** * Expand the given path/glob patterns into concrete script files. A pattern * that is an existing file passes through unchanged; anything else is treated * as a glob. Falls back to `defaultGlob` when no patterns are given. Results * are deduped and sorted. `ignore` is the repo's `ExcludeSet.globIgnore` * (src/exclude.ts — the floor plus `.vigilesrc.json#exclude`, correct from any * `cwd`; a plain list is relative to `cwd`), REQUIRED so a * vendored corpus's own `*.harness.mjs` cannot be discovered and run as ours * (#192). A path given explicitly in `patterns` is still run. */ export declare function discoverScripts(patterns: readonly string[], defaultGlob: string, cwd: string, ignore: GlobIgnore): string[]; /** Extra wiring for {@link runScripts}. */ export interface RunScriptsOptions { /** * How many scripts may run at once. Omitted → decided from `entry`: `test` * fans out across the cores, `eval` stays at 1. See {@link runScripts}. */ readonly concurrency?: number; /** * A program to run INSTEAD of each script, with the script's path as its one * argument. `vigiles eval` passes `dist/eval-entry.js`; `vigiles test` passes * nothing. See {@link interpreterArgs}. */ readonly entry?: string; } /** * Run each script as `node ` (or `node `, see * {@link RunScriptsOptions}), inheriting stdio so the script's own report * streams to the console. `env` is merged over `process.env` for every child * (e.g. `VIGILES_TRIALS`). Returns the per-file exit codes + check counts. * * Each child is handed its OWN scratch path in `VIGILES_CHECK_COUNT_ENV`, which * `vigiles` writes its check count to on exit — the channel that makes * "ran nothing" distinguishable from "ran and passed" (see check-count.ts). It * has to be a file: stdio is inherited so the script's report streams live, * which leaves no stream to parse. * * The same channel carries WHICH SURFACES the script exercised, so the caller can * record them (`.vigiles/coverage.json`) and coverage can answer "tested?" from * execution rather than from a matching file name. */ export declare function runScripts(files: readonly string[], cwd: string, env?: NodeJS.ProcessEnv, opts?: RunScriptsOptions): Promise; /** * Whether any script FAILED. Neither a skip nor a vacuous run counts: the first * declined to run, the second ran and verified nothing, and neither is evidence * that anything is broken. Both are visible in the summary instead. */ export declare function anyFailed(results: readonly ScriptRunResult[]): boolean; /** * What a `test`/`eval` invocation should do about actually RUNNING the discovered * scripts: * - `run` — proceed. * - `confirm` — interactive human, no explicit intent: ask before firing `count`. * - `refuse` — headless, no explicit intent: don't silently fire the whole tree. */ export type RunScriptsDecision = { readonly kind: "run"; } | { readonly kind: "confirm"; readonly count: number; } | { readonly kind: "refuse"; readonly count: number; }; export interface RunScriptsEnv { /** `test` is free/deterministic → always runs. `eval` spends model quota. */ readonly kind: "test" | "eval"; /** The user named explicit target files/globs (positional args) — clear intent. */ readonly explicitTargets: boolean; /** How many script files the discovery matched. */ readonly matchedCount: number; /** A human at a terminal who can answer + wait. */ readonly isTTY: boolean; /** `--all` — opt in to running the whole discovered set without a prompt. */ readonly all: boolean; /** `--yes` / `--no-interactive` — agent/CI mode: never prompt. */ readonly yes: boolean; /** * `--check` — VERIFY committed eval locks rather than measure. `decideLock` * in check mode returns only `replay` (the recorded report, no model call) or * `stale` (a failure), NEVER `run` — so this path cannot spend quota, and the * quota consent below must not stand in its way. * * Measured 2026-09-09: the CI `eval-check` step had never once executed. With * no lock committed anywhere, `eval --check` short-circuited on * `anyLocksCommitted` and returned "skip"; the first repo to commit a lock got * past that early return, reached this gate, and was refused exit 2. A gate * that is green because it never runs is the failure this repo keeps naming. */ readonly lockCheck: boolean; } /** * Consent gate for a bare (no-target) `vigiles eval`. `eval` runs the REAL model * on your subscription, and a no-target run discovers every `*.eval.*` over the * whole tree — so a repo with many evals fires them all and spends quota. Mirrors * `audit`'s read-vs-run consent (`decideExecute`): a paid, side-effecting verb * never fans out over an unbounded glob without either an explicit target, an * `--all` opt-in, or an interactive yes. `test` is free + deterministic, so it * always runs. Total + pure, first match wins; the IO (prompt/refuse) lives in the * CLI. */ export declare function decideRunScripts(o: RunScriptsEnv): RunScriptsDecision; /** One line per file + an explicit pass/skip/vacuous/fail tally. Skips and * vacuous runs are SHOWN, never folded into "passed" — a `⊘ SKIPPED` is loud, * not a silent green, and so is a file that verified nothing. */ export declare function formatScriptSummary(results: readonly ScriptRunResult[]): string; //# sourceMappingURL=run-scripts.d.ts.map