/** * Untested-surface detection: the third gap detector, alongside orphan-docs. * * Stale-ref detection catches specs pointing at files that vanished. * Orphan-docs catches docs nothing references. This catches harness *surfaces* * — skills, subagents, and hooks — that ship without a test or eval. A surface * with no test is a probabilistic-compliance gap hiding in the deterministic * layer: nothing measures whether it still does what it claims. * * TWO detectors decide "tested", in this order — **execution, then name, then * nothing** — and the report says which one answered: * * 1. EXECUTION — `.vigiles/coverage.json` records that a run exercised this * surface, at this version of it (`coverage-artifact.ts`). * 2. COLOCATION — a `*.{harness,eval}.mjs` named after the surface, beside it * (`skills/foo/foo.eval.mjs`, `hooks/pre-edit.harness.mjs`). * * Colocation was the only detector until 2026-08-11, and it answers a weaker * question than it appears to: it says a FILE EXISTS. `touch * .claude/skills/foo/foo.eval.mjs` — empty — drops the untested count by one. * No other coverage tool answers by name (`go test -cover`, coverage.py, nyc, * tarpaulin all answer from execution, using names only to FIND the file); we * kept the name because a skill cannot be run without a model, and a repo with * no runs recorded must behave exactly as it did before. * * A run recorded against an OLDER version of the surface grants nothing — it is * reported as `staleRuns` and the surface falls back to colocation. Silently * counting it is the PIPELINE-STATUS disease: a tick against a document that was * rewritten afterwards. * * There were three until 2026-08-11 (a `vigiles:covers` declaration, colocation, * and a content-reference "mention"). They were three NAMING CONVENTIONS, not * three strengths of evidence, and two of them could credit a surface no test * touched — measured on vigiles's own repo, `mention` supplied 9 of 10 covered * surfaces and at least three of those were false, including two hooks credited * by this detector's OWN test suite naming them as fixtures. See * `coverage-evidence.ts` for the full argument and the numbers. * * Colocation is kept because it cannot drift by construction: the test lives with * the surface, so deleting or renaming the surface takes its test along, and `ls` * answers "is this tested?" without running anything. * * Warning-by-default (a nudge, not a gate). EVERY skill, agent, and hook is held * to the requirement — invocation mode does NOT exempt anything (a command-only * skill still DOES something when invoked, and that behaviour is worth a test). * The only opt-out is explicit: a `vigiles:ignore-test` marker in the surface * file, which is reported as `exempt` so the skip is visible, never silent. * * TWO TIERS, discovered SEPARATELY — a harness and an eval are not the same thing * and collapsing them at discovery makes the difference unrecoverable downstream: * * | | harness (`*.harness.*`) | eval (`*.eval.*`) | * |---|---|---| * | cost | free | paid model calls | * | cadence | every push | scheduled | * | answers | "does this gate still catch what it claims?" | "does this skill FIRE at all?" | * * So a repo with complete deterministic coverage and no evals is a DIFFERENT * position from a repo with neither, and "add a test/eval" spans three orders of * magnitude in cost without saying which. {@link UntestedReport} therefore carries * a per-tier {@link CoverageTier} alongside the (unchanged) union fields. */ import { type ExcludeSet } from "./exclude.js"; import type { RepoPath } from "./core/frame.js"; import { type PluginLayout } from "./core/layout.js"; import { type CoverageEvidence, type EvidenceCounts } from "./coverage-evidence.js"; export type SurfaceKind = "skill" | "agent" | "hook" | "output-style"; export interface Surface { readonly kind: SurfaceKind; /** Repo-relative path to the surface file (SKILL.md / agent .md / hook script). */ readonly path: string; /** Stable name: skill dir, agent basename, or hook script basename. */ readonly name: string; /** Substrings a test may reference to "cover" this surface (path / namespace). */ readonly tokens: readonly string[]; /** Explicitly opted out of the test requirement via `vigiles:ignore-test`. */ readonly ignored: boolean; } /** * WHY one surface counts as covered — the provenance of a single decision. * Reported so a repo can see what its coverage number actually rests on; a count * whose derivation is invisible is exactly the failure this detector had. */ export interface CoverageDecision { readonly surface: Surface; readonly evidence: CoverageEvidence; /** The test file the (strongest) evidence came from. */ readonly by: string; } /** A surface measured by a run that no longer describes it. */ export interface StaleRun { /** Repo-relative path of the surface. */ readonly path: string; /** The script whose run measured the older version. */ readonly by: string; /** When that run happened (ISO-8601). */ readonly at: string; } /** A test file whose NAME is why it does not count — see `retiredTestNames`. */ export interface RetiredTestName { /** Repo-relative path of the file that will not count. */ readonly path: string; /** Repo-relative path of the untested surface it sits beside. */ readonly surface: string; } /** One tier's split of the considered surfaces — covered by THAT tier, or not. */ export interface CoverageTier { readonly covered: readonly Surface[]; readonly untested: readonly Surface[]; /** One entry per `covered` surface, in the same order — how it was decided. */ readonly decisions: readonly CoverageDecision[]; } export interface UntestedReport { /** Total surfaces considered (after exemptions). */ readonly total: number; /** Covered by EITHER tier — the union, unchanged (a test anywhere counts). */ readonly covered: readonly Surface[]; /** Covered by NEITHER tier — the union, unchanged. */ readonly untested: readonly Surface[]; /** Surfaces explicitly opted out via `vigiles:ignore-test`. */ readonly exempt: number; /** * The exempt surfaces themselves. The marker waives a surface's test, not its * existence, so an inventory counts them (an exempt style still ships). */ readonly exemptSurfaces: readonly Surface[]; /** * Test files still carrying the RETIRED `vigiles:covers` marker. * * 🔴 A MIGRATION THAT WOULD OTHERWISE BE SILENT. Up to 15.0.2 this tool's own * untested finding told the reader to "mark what it covers with * `vigiles:covers `". That tier is gone: coverage is decided by where * a test sits and what it is named. Someone who followed the instruction gets * no error on upgrade — the marker simply becomes a comment, their coverage * drops and the count of untested surfaces rises with nothing said about why. * * Reading these files does NOT feed coverage (nothing inside a test can change * it any more); it exists purely so the upgrade can explain itself. Optional * so a report produced before this field still parses. */ readonly legacyCoversFiles?: readonly string[]; /** * Files sitting beside an UNTESTED surface, named after it, and carrying a * suffix a default vitest/jest run collects — `.test.*`, * `.spec.*`. * * 🔴 THE OTHER HALF OF THE SAME 15.x MIGRATION, and it was the silent one. * `vigiles:covers` got a note; `*.test.*` leaving {@link DEFAULT_TEST_GLOBS} * did not, on the reasoning recorded there that "the migration is a rename, * and the untested finding prints the exact path". The path it prints is the * SURFACE's, plus a suggestion to add `.eval.mjs` — so an author * looking at `foo.test.mjs` lying right next to `foo/SKILL.md` is told to * write a test they already wrote, and nothing names the file or the reason. * * Scoped to surfaces that are otherwise UNCOVERED, which is exactly the * position where the count contradicts what the author can see. A stray * `*.test.*` beside a properly covered surface is somebody's ordinary unit * test and none of our business. Optional so an older report still parses. */ readonly retiredTestNames?: readonly RetiredTestName[]; /** Extension a generated test should use — see `core/test-file-ext.ts`. * Optional so a report produced before this field still parses. */ readonly testExt?: string; /** * How each covered surface was decided — the union tier's provenance, one * entry per `covered` element. A coverage count whose derivation is invisible * is what let a COMMENT confer coverage unnoticed; this is the visibility. */ readonly decisions: readonly CoverageDecision[]; /** * Surfaces whose ONLY run record predates their current text — "measured, but * not this version". * * They grant no coverage (they fall back to colocation like anything else), * because a measurement of a file that has since been rewritten is a * measurement of a different file. Reported rather than dropped, since the * silent version of this is the PIPELINE-STATUS failure mode: a green tick * against a document somebody edited afterwards. Optional so a report produced * before this field still parses. */ readonly staleRuns?: readonly StaleRun[]; /** * DETERMINISTIC coverage only — `*.harness.*`, plus any custom `include`. * Free, millisecond, every-push. Answers "does this gate still catch what it * claims?" (`*.test.*` has NOT counted since 15.x — see DEFAULT_TEST_GLOBS.) */ readonly harness: CoverageTier; /** * REAL-MODEL coverage only — `*.eval.mjs`. Paid, minutes, scheduled. Answers * the one question the deterministic tier cannot: "does this skill FIRE at all?" */ readonly evals: CoverageTier; } export interface TestCoverageOptions { /** The bundle whose surfaces are discovered. Required: no cwd fallback. */ readonly basePath: string; /** * The directory `include`, the coverage artifact and every REPORTED * path are relative to — where `.vigilesrc.json` lives. Defaults to `basePath`. * Differs from it for a nested bundle under `bundles: "all"` (#281). */ readonly root?: string; /** Scan skills under `skills/` and `.claude/skills/`. Default true. */ readonly skills?: boolean; /** Scan subagents under `agents/` and `.claude/agents/`. Default true. */ readonly agents?: boolean; /** Scan hook scripts referenced from plugin.json / settings.json. Default true. */ readonly hooks?: boolean; /** * Scan output styles, in every folder the layout's `outputStyles` names. * Default true; a layout with no output styles finds none either way. */ readonly outputStyles?: boolean; /** * Every harness the repository declares, when it declares more than one. * Surfaces are found in each of them, one per file: a repository can list a * harness first and keep its skills, agents or styles in another's folder. * Defaults to `[layout]`. */ readonly harnessLayouts?: readonly PluginLayout[]; /** Globs of test files that count as coverage. */ readonly include?: readonly string[]; /** * Which extension a GENERATED test gets. Detection (a tsconfig.json, a * typescript dependency) decides by default; this field exists only to * disagree with it. Deliberately NOT written by `init`: a value recorded at * initialisation goes stale in silence when the project migrates, while * detection re-runs every time. * * From `.vigilesrc.json` it arrives as `rules["untested-skill"][1].testExtension` * (or the `-subagent` / `-hook` twin — the three share their options). The * previous wording here said only "from `.vigilesrc.json`", which read as a * promise the CLI did not keep: `TestCoverageConfig` had no such key and * `checkUntestedSurfaces` forwarded only `include`/`exclude`, so a configured * `mjs` was silently ignored on any TypeScript-shaped repo. */ readonly testExtension?: string; /** * The repo-wide `.vigilesrc.json#exclude`. It carries its own root, so it is * applied correctly whichever directory is being discovered — a string list * here was correct only when that directory WAS the repo root (#281). */ readonly excludes?: ExcludeSet; /** * Harness layout — where skills/agents live, the plugin-root token, the * manifest/settings paths. Defaults to Claude Code; a non-CC adapter passes its * own so the surface globs and hook-token expansion aren't hard-coded. */ /** * The layout to discover surfaces under. REQUIRED: it used to default to the * Claude Code layout, which meant this harness-agnostic detector imported an * adapter, and a caller that forgot to pass one silently graded a Codex repo * with Claude Code's directories. */ readonly layout: PluginLayout; } /** * Find harness surfaces (skills / agents / hooks) that no test or eval covers. * A surface is covered by a colocated `*.{harness,eval}.mjs` OR any discovered * test that references its path/namespace. EVERY skill, agent, and hook is held * to this — the only exemption is an explicit `vigiles:ignore-test` marker in the * surface file (counted as `exempt`). * * The `covered`/`untested` union fields are UNCHANGED (a test anywhere counts). * `harness` and `evals` carry the per-tier split so a caller can tell "has * deterministic coverage, no evals" from "has neither" — two positions the single * count made indistinguishable. */ export declare function findUntestedSurfaces(options: TestCoverageOptions): UntestedReport; /** Suggested colocated test path for an untested surface (shown in the warning). */ export declare function suggestedTestPath(surface: Surface, ext?: string): string; /** Tally the union tier's coverage decisions by how each was established. */ export declare function coverageEvidenceCounts(report: UntestedReport): EvidenceCounts; /** * The edit-time half of `untested-skill` — the nudge a PostToolUse hook delivers * when the agent has just edited a skill/agent surface. * * WHY it lives next to `findUntestedSurfaces` instead of being its own detector: * the rule was already stated correctly, but it only ran inside `vigiles lint`, * which a human has to remember to type. A rule that fires only when invoked by * hand is prose, not policy — so this reuses the SAME detector at the moment the * surface changes. * * It deliberately does NOT say "write a test". An agent already knows that; what * it did not know is **with what** — so the message's job is to hand off to the * `test-harness` skill, which carries the tier→API table. A nudge that restates * the obligation and withholds the vocabulary is the failure this replaces. * * Two distinguishable gaps, because they cost orders of magnitude apart: * - no test at all → start at the cheapest tier * - a harness but no eval → you just changed the TRIGGER surface, and a * deterministic harness structurally cannot tell you * whether a description still fires * * Returns `null` when the edited file isn't a skill/agent surface, or when it is * covered on both tiers. Never throws — a nudge must not disrupt an edit. */ export declare function skillTestNudge(filePath: RepoPath, options: TestCoverageOptions): string | null; /** * What the PAID eval tier measures for a surface of this kind — or `null` when * it measures nothing for it. * * 🔴 THE REMEDY USED TO BE ONE SENTENCE, AND IT WAS THE SKILL'S. Every * never-evaluated surface was told its "description must FIRE" and pointed at * `measureTriggerRate`. An agent reaches that branch as readily as a skill — * `skillTestNudge` disables only hooks — and for an agent the sentence is false * twice over: firing is not the agent's eval-tier question, and * `measureTriggerRate` cannot address an agent at all. * * MEASURED 2026-08-12, no model spent, against this build: * * ``` * packageSkillsDir("/agents") → THREW: No /SKILL.md skills * found under /agents * measureTriggerRateWith({ pluginDir: }, fakeRunner) * ``` * * The loose-skills form throws; the plugin form returns a NUMBER for a surface * it never installed. That is the same failure the driver-argument note one * branch up already calls out — worse than an error, because it looks like a * measurement — and following the old wording produced it. * * ## Why this is keyed on `kind`, and what happens to a fourth one * * The real predicate is a CAPABILITY — "can the trigger tier install and observe * this surface?" — and today that is a total function of `kind`, because the * tier's installer is `/SKILL.md`-shaped and its `fired` predicate is skill * SELECTION (`skillResolved`). So the capability is stated once per kind here, * rather than inferred from a name at the call site. * * The switch is exhaustive over {@link SurfaceKind} with `assertNever`: a fourth * kind does not silently inherit the skill's sentence, it fails to COMPILE until * someone writes what the eval tier measures for it. If the trigger tier ever * grows an agent installer, this table is the single place that changes. */ export declare function evalTierQuestion(kind: SurfaceKind): string | null; /** * The QUALIFIERS on a coverage number — every caveat that says "this count is * not quite what it looks like", in one list, built once. * * 🔴 THIS EXISTS BECAUSE A CAVEAT COULD BE PRINTED BY ONE COMMAND AND NOT THE * OTHER, AND WAS. Both notes below were added to `formatUntestedReport` (the * `lint` renderer) and neither reached `audit`, which assembles its own fact * block from the same {@link UntestedReport}. Measured 2026-08-18 on a fixture * whose only harness carried the retired `vigiles:covers` marker: `lint` named * the file, `audit` printed `Untested surfaces: 0` and nothing else. The * migration note existed, was unit-tested, and was invisible to anyone whose * habit is `audit` — which is the whole point of a note that explains a silent * migration. * * Collecting them here is the subtraction: a caveat is no longer something a * renderer can choose to carry. Adding a third one reaches both callers or * neither, and "neither" is a compile error rather than a quiet omission. */ export declare function coverageCaveats(report: UntestedReport): readonly string[]; export declare function formatUntestedReport(report: UntestedReport): string; //# sourceMappingURL=test-coverage.d.ts.map