import { type RunEnv } from "./core/run-env.js"; import type { HarnessLiveDriver } from "./core/live-driver.js"; import type { AgentRunArgs, AgentRunner, EvalDriver, EvalUsage, ModelOutputParser, ParsedModelRun, RunOut } from "./core/eval-driver.js"; export type { AgentRunArgs, AgentRunner, EvalDriver, EvalUsage, ModelOutputParser, ParsedModelRun, RunOut, } from "./core/eval-driver.js"; import { type ToolCall, type Trace } from "./harness-test.js"; import { type CacheMode } from "./eval-cache.js"; import { type EvalLockOptions } from "./eval-lock.js"; import type { Check, CheckJSON } from "./check.js"; import { type Comparison } from "./stats.js"; import { type ToolIntercept } from "./tool-intercept.js"; import { type StubCall, type ToolStub } from "./core/stub-rules.js"; /** One arm of the comparison: fixture overrides + settings (hooks) for this arm. */ export interface EvalArm { /** Files written on top of the base fixture for this arm. */ readonly files?: Record; /** `.claude/settings.json` (hooks/permissions) for this arm; omit for none. */ readonly settings?: unknown; /** * Path to a real plugin/repo to load for this arm (hooks + CLAUDE.md + * skills). Lets an arm be "the whole plugin on" vs "off". See * src/plugin-loader.ts. */ readonly plugin?: string; /** * Path to a plugin dir to install NATIVELY (`claude --plugin-dir`) for this * arm, so its skills/commands/agents activate the real way — the real model * can trigger a skill by its description (vs. `plugin`, which materializes a * file subset that does not register skills). Point at a COMPLETE plugin. Lets * an arm be "skill installed" vs "off" to measure real activation. Provide this * OR {@link skillsDir}, not both. */ readonly pluginDir?: string; /** * A directory of LOOSE skills (`//SKILL.md`, e.g. a repo's * `.claude/skills`) to install for this arm. vigiles packages them into a * throwaway `--plugin-dir` (manifest + `skills//`, each skill dir copied * whole) for the run and removes it afterward — the one-liner for repo-local * skills that aren't a published plugin. The skills install under the * namespace `vigiles-loose-skills`, so a `skill()` check / `skillResolved` * matches `vigiles-loose-skills:` (the report's `namespace` says so). * Provide this OR {@link pluginDir}, not both. */ readonly skillsDir?: string; /** * Tools to intercept for this arm (the tool-call spy). Each * {@link ToolIntercept} is denied its real execution by an auto-wired PreToolUse * hook — the model still emits the `tool_use` (so its arguments land in the * `Trace` for `toolWith` / `notTool`), but the side effect (a paid API call, a * `git push`, a paid subagent) never happens: the call is intercepted * (prevented), NOT executed. This makes a real-model eval **side-effect-free and * safe** (it does NOT cut the model-call cost); and because CC surfaces the * denial as a *blocked* call, it's for asserting the agent's ATTEMPT, not for * stubbing a tool to continue a multi-step flow. See `src/tool-intercept.ts`. */ readonly interceptTools?: readonly ToolIntercept[]; /** * Model alias/id for THIS arm, overriding the eval-level `model`. A model * comparison IS a harness A/B — `arms: { sonnet: { model: "claude-sonnet-4-6" }, * opus: { model: "claude-opus-4-8" } }` — so model-as-an-arm answers "does my * harness still hold on the cheaper tier / after a model upgrade?" through the * same significance machinery, with no separate model-matrix runner. Omit to * use the eval-level model. See `research/eval-architecture.md` (model strategy). */ readonly model?: string; /** * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer). * Lives here beside `model` because it is part of the MEASUREMENT — it moves the * output distribution, not the sample size — so it is hashed into the lock and * the cache, and never read from an env var. Omit for the harness default. */ readonly effort?: string | number; } /** * Context handed to `measure` after a run, to compute that run's metrics. It is * a `Trace` (so the bare predicates `usedTool` / `skillResolved` / `toolCount` / * `toolUsedWith` from `harness-assert.ts` run over it, the same as over a * `runHarnessTest` result) plus the eval-only `sh` end-state probe and `usage`. */ export interface RunContext extends Trace { readonly cwd: string; readonly exitCode: number; readonly stdout: string; /** `num_turns` reported by claude, or 0. */ readonly turns: number; /** The tools the agent invoked, each paired with its result (parsed from the stream). */ readonly toolCalls: readonly ToolCall[]; /** Cost / latency / tokens for this run (use as metrics, e.g. `{ cost: ctx.usage.costUsd }`). */ readonly usage: EvalUsage; /** Run a shell command in the working dir; returns trimmed stdout ("" on error). */ sh(command: string): string; } export type Metrics = Record; export interface EvalSpec { readonly name?: string; /** Base fixture files (path → contents), written fresh for every run. */ readonly fixture?: Record; /** The arms to compare, by name. */ readonly arms: Record; /** * Stub each arm's skill BODIES (frontmatter kept) before the run — for firing * comparisons (does the skill get SELECTED?), where a selected skill should * stop at selection instead of running its (often expensive) procedure. Every * arm with a `pluginDir` / `skillsDir` is repackaged with bodies stripped; arms * without one are untouched. Don't combine with quality metrics: the body is * gone, so there is nothing to grade. See {@link stubSkillBody}. */ readonly stubSkillBodies?: boolean; /** The task prompt given to the agent. */ readonly task: string; /** Compute this run's metrics from its outcome. */ readonly measure: (ctx: RunContext) => M; /** Trials per arm. Default 5. */ readonly trials?: number; /** Model alias. Default "haiku". */ readonly model?: string; /** * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer). * Lives here beside `model` because it is part of the MEASUREMENT — it moves the * output distribution, not the sample size — so it is hashed into the lock and * the cache, and never read from an env var. Omit for the harness default. */ readonly effort?: string | number; /** * Tools pre-approved for the run (`--allowedTools`, a permission allowlist); * the default tools stay available either way. `[]` approves none. * Default: Read Edit Write Bash. */ readonly allowedTools?: readonly string[]; /** Per-run timeout ms. Default 240000. */ readonly timeoutMs?: number; /** Seconds to wait between runs (avoid rate-limit bursts). Default 4. */ readonly spacingSec?: number; /** * Record/replay cache mode. Default `"off"` (always re-sample). `"readwrite"` * records each trial (output + post-run files) and replays it on a matching * re-run — so editing `measure` re-scores for free; the model is re-called only * when a model-affecting input changes. `"read"` replays but never records. * The cache key excludes `measure`, so changing your metric still hits. */ readonly cache?: CacheMode; /** Where cache records live. Default `.vigiles/eval-cache` under cwd. */ readonly cacheDir?: string; /** * How many trials to run at once (across all arms × trials). Default 1 (fully * sequential — the safe, no-surprise default). Raise it to cut wall-clock time; * rate-limit bursts are absorbed by the retry/backoff below. */ readonly concurrency?: number; /** * Abort the run once measured cost reaches this many USD. In-flight trials * finish; remaining ones are skipped and `report.aborted` is set. Needs the * model to report `total_cost_usd` (the eval tier does). */ readonly maxCostUsd?: number; /** Retries on a detected rate-limit/overload before giving up. Default 3. */ readonly rateLimitRetries?: number; /** Base backoff ms (doubled each retry). Default 1000. */ readonly retryBackoffMs?: number; /** * **Where each trial runs — required, no default.** * * - `{ kind: "ephemeral" }` — a throwaway HOME (under the trial's temp dir) and * a scrubbed environment: the OS essentials, the harness's own auth * (`runEnv.keep`, plus its auth file copied in), the eval's own `VIGILES_*` * variables, and nothing else. `home: { kind: "files", files }` seeds that * HOME first (HOME-relative paths; the harness's auth file cannot be * overwritten). Running a model-driven skill is itself a side effect, so a * `git push` or a write to `~` lands in a disposable HOME. * - `{ kind: "inherit", reason }` — your real HOME and environment, minus the * parent session's identity. `reason` is printed with the report. * * There is no default because both are wrong silently in one case: inheriting * measures the author's machine (their config, their tokens), and an * ephemeral run fails if a user's credential is not where the harness says it * is — loudly, on trial 1, which is why it is the one to try first. See * `docs/safety.md` (what a child run inherits). */ readonly env: RunEnv; /** * **Tool stubs on PATH (rung R2)** — fake binaries that answer PER INVOCATION. * Build each with {@link experimental_stub}: a list of rules over the argv * tokens, first match wins. vigiles writes a shim per stub into a temp dir * BESIDE the trial's working dir and puts it first on the run's PATH, so a * skill/hook/agent — or a script it runs — that calls `gh`/`psql`/`git` gets * the recorded answer instead of the real service. * * An invocation no rule answers is UNANSWERED: the model sees one neutral * stderr line and a non-zero exit, no new trials start, the report lists the * call (`ArmReport.unansweredStubCalls`), `vigiles eval` exits 2 and * `--update` writes no lock. Every call is recorded in `RunContext.stubCalls`, * so `experimental_stub.called(...)` can assert one happened. * * The answers are author/recorded fixtures, **never** model-synthesized — a * synthesized tool output looks plausible but diverges from the real tool. */ readonly stubs?: readonly ToolStub[]; /** * **The eval LOCK** — a committed staleness gate for CI (see `src/eval-lock.ts`). * With a `name` set, `vigiles eval --update` (local, on your subscription) * records the report to `.vigiles/eval-locks/.lock.json`; `--check` (CI) * verifies the committed result against the current inputs WITHOUT a model call, * failing "stale" when they diverge. Mode normally comes from the CLI; set * `lock` to drive it programmatically or point a test at a throwaway dir. The * lock engages only when `name` is set (it keys the lock file). */ readonly lock?: EvalLockOptions; } /** Per-metric summary statistics across an arm's runs. */ export interface MetricStat { /** Mean (numbers) / fraction-true (booleans). */ readonly mean: number; /** Sample standard deviation (0 when n < 2). */ readonly std: number; /** Standard error of the mean (std / √n). */ readonly se: number; /** Number of runs the metric was observed in. */ readonly n: number; /** * pass^k (τ-bench): 1 if the metric succeeded on EVERY trial, else 0. The * reliability question a non-deterministic harness needs — "worked every time" * is not "worked on average". A trial counts as a success when its value is * truthy (booleans true, counts > 0), so model your metric as success/fail. */ readonly passK: number; } /** Aggregated cost / latency / tokens across an arm's runs. */ export interface ArmUsage { readonly totalCostUsd: number; readonly meanCostUsd: number; readonly meanDurationMs: number; readonly totalInputTokens: number; readonly totalOutputTokens: number; readonly totalCacheCreationTokens: number; readonly totalCacheReadTokens: number; } export interface ArmReport { readonly runs: number; /** Aggregated metrics: mean for numbers, fraction-true (0..1) for booleans. */ readonly metrics: Record; /** Per-metric mean / std / se / n, so an A/B gap can be read for significance. */ readonly stats: Record; /** Cost / latency / token totals + means for this arm. */ readonly usage: ArmUsage; /** * Stub invocations no rule answered, across this arm's trials. Non-empty means * the arm measured at least one answer nobody wrote: `vigiles eval` exits 2 on * it and `--update` refuses to record the lock. Empty when the eval has no stubs. */ readonly unansweredStubCalls: readonly StubCall[]; } export interface EvalReport { readonly name: string; readonly trials: number; readonly arms: Record; /** Total measured cost across every arm × trial (0 when usage wasn't reported). */ readonly totalCostUsd: number; /** True if a `maxCostUsd` budget cap stopped the run before all trials ran. */ readonly aborted: boolean; /** * The run environment the trials used, without seeded file contents — so an * `inherit` run's reason travels with the report and is printed with it. * Absent on a report built before `env` existed. */ readonly env?: { readonly kind: "ephemeral"; } | { readonly kind: "inherit"; readonly reason: string; }; } /** * Resolve the environment a trial's subprocess actually runs with — the * SECURITY-CRITICAL decision behind `env: { kind: "ephemeral" }`. When `replaceEnv` is set, the * scrubbed `env` is the COMPLETE environment, so the real `$HOME` and inherited * secrets are DROPPED; otherwise `env` is an overlay on `base`. In BOTH, the * harness's declared session identity (`runEnv.sessionIdentity`) is removed: a * run started from inside a live session must not run as that session, scrubbed * or not. Extracted from the `v8 ignore`d `spawnAgent` so the one line that * enforces the scrub is both unit- and behaviourally-tested — a regression to an * always-merge would otherwise silently defeat ephemerality and leak the host * environment into an untrusted, model-driven run. */ export declare function resolveSpawnEnv(a: Pick, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv; /** * The env var name the harness reads for the reasoning budget. It sits ABOVE the * `--effort` flag in the CLI's own precedence chain, so passing the flag alone * does NOT pin the level. */ export declare const EFFORT_ENV_VAR = "CLAUDE_CODE_EFFORT_LEVEL"; /** * Pin the effort the run actually gets, so the recorded effort is the effort * that ran. * * WHY THIS EXISTS AND WHY IT IS NOT OPTIONAL. Effort has THREE inputs — the * `--effort` flag, the `effortLevel` settings key, and `CLAUDE_CODE_EFFORT_LEVEL` * — and the env var wins over the flag. The default (inherited) spawn env * carries the caller's whole environment, so an ambient * `CLAUDE_CODE_EFFORT_LEVEL=max` in the author's shell reaches the run unless * something removes it. Without this pin, * hashing effort into the lock would make the lock CONFIDENTLY WRONG: it would * record `low` over a run that executed at `max` — the exact defect the feature * exists to prevent, reintroduced by the fix for it. * * Both directions matter, so both are handled: * - effort DECLARED → set the var, overriding whatever the shell had. * - effort OMITTED → DELETE an inherited var, so "omit" means the harness * default rather than "whatever this machine happened to * export". An omitted effort must not be a hidden input. */ export declare function pinEffortEnv(env: NodeJS.ProcessEnv, effort: string | number | undefined): NodeJS.ProcessEnv; /** * The harness's own rejection of an `--effort` value, or null. Pure. * * The CLI does NOT fail on a bad level — it prints this to stderr and silently * runs at its default. That silent substitution is precisely the bug class this * feature addresses (a number produced by a configuration nobody asked for), so * a rejected value must never become a sample. Matched on the binary's own * wording, the same shape as {@link isRateLimited}. */ export declare function effortRejection(out: RunOut): string | null; /** * Wrap a runner so a run the harness rejected on `--effort` FAILS LOUDLY. * * Applied ONCE, around the real runner, rather than as a guard repeated at each * of the five `runner(...)` call sites — a guard per call site is the shape that * left four of five compilers unprotected in #173. * * It THROWS rather than counting the trial as `runError`. A `runError` trial is * dropped from the denominator, which is right for a transient (a rate limit) and * wrong here: an unusable effort value is deterministic and repeatable, so every * trial fails it and the run would report a rate computed over ZERO samples. A * configuration mistake should stop the run and name itself. */ export declare function withEffortGuard(runner: AgentRunner): AgentRunner; /** * Build the real runner's argv. Pure and exported so the FLAGS are provable — * `spawnAgentRaw` is `v8 ignore`d (it spawns a subprocess), so an argv assembled * inline there could not be asserted at all. Mirrors `buildCodexArgs`. */ export declare function buildAgentArgs(a: AgentRunArgs): string[]; /** * Why a run never reached the model, or null when it did. A run that exited * non-zero with no `assistant` event and no `result` event asked the model * nothing — a usage error in the command line, a missing binary — and scored as * a trial it reads as a miss on every check that needs output and a pass on * every bound it cannot break. A run that reached the model and then broke is a * scored outcome, not this. Rate-limited runs are left to the retry loop. */ export declare function startFailure(out: Readonly): string | null; /** * Wrap a runner so a run that never reached the model THROWS instead of being * scored — the same choice, for the same reason, as {@link withEffortGuard}: * the cause is usually deterministic, so every trial would fail it. Not `async`, * for the reason that guard gives. */ export declare function withStartGuard(runner: AgentRunner): AgentRunner; /** * The real `claude`-spawning runner (composition root). Exported so other * real-model entries (e.g. the `audit` trigger tier) bind the same runner. * * The effort guard is composed in HERE, at the single definition, rather than at * each of the places that bind this runner — so every consumer, including ones * not yet written, is covered by construction. Guarding each call site instead is * the shape that left four of five compilers unprotected in #173. */ export declare const spawnAgent: AgentRunner; /** * Run the eval: every arm × every trial against the real `claude` CLI, with the * metric computed per run and aggregated per arm. Requires `claude` on PATH and * working model auth (e.g. `ANTHROPIC_API_KEY`). Thin wrapper over * `runEvalWith` with the real agent runner. */ /** * A `SKILL.md` written straight into a run's cwd (via an arm's `files`) is NOT * registered as a skill by the harness. Claude Code — and Codex — load skills only * from a plugin / `.claude/skills` layout, so a bare cwd `SKILL.md` sits unread: * the arm silently measures NOTHING (baseline and "skill" become the same run). This * is the exact footgun the ecosystem benchmark hit — a skill file delivered where it * can never activate, with no error. Detect it so {@link runEval} / {@link measureArms} * can WARN and point at `pluginDir` (a real `--plugin-dir` install) or `skillsDir`. * * HIGH-PRECISION (don't cry wolf): only a file whose basename is `SKILL.md` AND that * carries real skill frontmatter (a `---` block naming `name`/`description`) is flagged * — so an empty scratch `SKILL.md` a task is asked to AUTHOR is never flagged. Pure + * exported for testing. */ export declare function unregisteredSkillFiles(files: Record | undefined): string[]; export declare function runEval(spec: EvalSpec): Promise; /** A task run N times, scored against a `Trace` check vocabulary. */ export interface MeasureSpec { /** Base fixture files written for every run (path → contents). */ readonly fixture?: Record; /** `.claude/settings.json` (hooks/permissions) for the run. */ readonly settings?: unknown; /** A real plugin/repo to load (materialized) — see `EvalArm.plugin`. */ readonly plugin?: string; /** * A complete plugin dir to install natively (`--plugin-dir`) so skills activate * — see {@link EvalArm.pluginDir}. Provide this OR `skillsDir`, not both. */ readonly pluginDir?: string; /** * A loose `//SKILL.md` directory (e.g. a repo's `.claude/skills`) * to install, auto-packaged into a throwaway plugin — see {@link EvalArm.skillsDir}. * Installs under `vigiles-loose-skills`, so `skill("vigiles-loose-skills:")`. */ readonly skillsDir?: string; /** * Stub each skill BODY (frontmatter/trigger surface kept) before the run — for * checks about whether a skill FIRES (`skill()`), not what it produces. A * selected skill stops at selection instead of running its (often expensive) * procedure, so a description/firing run costs a fraction of the tokens. Do NOT * combine with `judged`/quality checks: the body is gone, so there's nothing to * grade. Requires `pluginDir` or `skillsDir`. See {@link stubSkillBody}. */ readonly stubSkillBodies?: boolean; /** Tools to intercept (the tool-call spy) — see {@link EvalArm.interceptTools}. */ readonly interceptTools?: readonly ToolIntercept[]; /** The task prompt given to the agent. */ readonly task: string; /** * The checks to score across the trials. Any check over the run is accepted: * `Trace` checks (`tool`/`skill`/`output`/`mcp`/`judged`) and resource checks * (`cost`/`latency`/`tokens`, which read the eval-only `usage`) — all fit * `Check`. */ readonly checks: readonly Check[]; /** Trials. Default 5. */ readonly trials?: number; /** Model alias. Default "sonnet" — measure on the model your users run. */ readonly model?: string; /** * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer). * Lives here beside `model` because it is part of the MEASUREMENT — it moves the * output distribution, not the sample size — so it is hashed into the lock and * the cache, and never read from an env var. Omit for the harness default. */ readonly effort?: string | number; /** Tools pre-approved for the run (`--allowedTools`); `[]` approves none. */ readonly allowedTools?: readonly string[]; /** Per-run timeout ms. */ readonly timeoutMs?: number; /** Seconds between runs. */ readonly spacingSec?: number; } /** One check's measured rate across the trials. */ export interface CheckRate { readonly check: CheckJSON; /** Fraction of trials the check passed (0..1). */ readonly rate: number; /** Standard error of the rate. */ readonly se: number; /** pass^k — 1 iff the check passed on EVERY trial. */ readonly passK: number; /** Trials observed. */ readonly n: number; } export interface CheckReport { readonly n: number; readonly perCheck: readonly CheckRate[]; /** Cost / latency / token totals for the run (the same source as `runEval`). */ readonly usage: ArmUsage; /** * The plugin namespace the skills actually installed under — the `` * half of the `:` id a `skill()` check matches. Undefined when * the run installed nothing (no `pluginDir` / `skillsDir`). Reported because * with `skillsDir` the name is chosen by the packager, not the caller, so the * most common cause of a 0% `skill()` rate was a value the caller never saw. * Mirrors {@link TriggerRateReport.namespace}. */ readonly namespace?: string; } /** * Score a check vocabulary across trials — the scored counterpart to * `assertChecks` (strict). Each check yields a `rate ± se` and `pass^k` over `n` * runs. Reuses the tested `runEvalWith` aggregation (one arm), so the loop, * cache, concurrency, and stats come for free. Exported with an injectable * `runner` so the orchestration is unit-testable without a model. */ export declare function measureWith(spec: MeasureSpec, runner: AgentRunner): Promise; /** Score a check vocabulary across trials against the real `claude` CLI. */ export declare function measure(spec: MeasureSpec): Promise; /** A task scored against checks across NAMED arms (the harness variable on/off). */ export interface ArmsMeasureSpec { readonly fixture?: Record; /** The arms to compare (settings / plugin / pluginDir per arm). */ readonly arms: Record; readonly task: string; readonly checks: readonly Check[]; /** * Stub each arm's skill BODIES (frontmatter kept) before the run — the A/B * counterpart to {@link MeasureSpec.stubSkillBodies}. For firing comparisons * (does description variant A fire more than B?), every arm that sets * `pluginDir` / `skillsDir` is repackaged with bodies stripped so each run * stops at selection — a fraction of the tokens. Arms without one are left * untouched. Don't combine with `judged`/quality checks. See {@link stubSkillBody}. */ readonly stubSkillBodies?: boolean; readonly trials?: number; readonly model?: string; readonly allowedTools?: readonly string[]; readonly timeoutMs?: number; /** * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer). * Lives here beside `model` because it is part of the MEASUREMENT — it moves the * output distribution, not the sample size — so it is hashed into the lock and * the cache, and never read from an env var. Omit for the harness default. */ readonly effort?: string | number; readonly spacingSec?: number; } /** Per-arm {@link CheckReport}s — `arms[name].perCheck[i]` aligns across arms. */ export interface ArmsCheckReport { readonly arms: Record; } /** Score checks across arms (injectable runner). Reuses `runEvalWith`. */ export declare function measureArmsWith(spec: ArmsMeasureSpec, runner: AgentRunner): Promise; /** Score checks across arms against the real `claude` CLI. */ export declare function measureArms(spec: ArmsMeasureSpec): Promise; /** * Welch significance on one check's rate between two arms (`arm` vs `baseline`), * by index in `perCheck`. So "the gated arm resolves the skill significantly more * than vanilla" is a p-value, not a vibe. Reuses `welchTTest` from stats.ts. */ export declare function compareCheck(report: ArmsCheckReport, baseline: string, arm: string, checkIndex: number): Comparison; /** * Format a {@link CheckReport}: one line per check with its rate ± se and * pass^k, and the line saying `measure` ran with your HOME and environment * (it has no `env` field; `INHERITED_BY_MEASURE`). */ export declare function formatCheckReport(report: CheckReport): string; /** * Format an {@link ArmsCheckReport}: each arm's checks under `[arm: name]`, and * the inherit line ONCE at the end — every arm ran the same way. */ export declare function formatArmsCheckReport(report: ArmsCheckReport): string; /** * The scored gate (Phase 4): throw if any check's measured rate is below its * threshold — the `measure` counterpart to `assertChecks` (strict). Reads the * rate, not a single run, so it never trips on one noisy trial. * * `min` is the default threshold for every check. `per` overrides it for a check * KIND (e.g. `{ min: 0.8, per: { skill: 1.0 } }` — "every check ≥ 80%, but the * skill must FIRE on every trial"), so a strict firing/safety check and a * lenient quality check gate in one call — the single-skill absolute oracle * (`measure({ checks: [skill(), judged()] }) + assertRates`) needs exactly this. */ export declare function assertRates(report: CheckReport, opts: { min: number; per?: Readonly>; }): void; /** * Serialize a {@link CheckReport} to JUnit XML (Phase 4) — each check a * ``, failing when its rate is below its threshold. `min` is the * default; `per` overrides it by check KIND, matching `assertRates` exactly (one * shared threshold helper) so the gate and the report can never disagree about * which checks failed. Because a check is *data*, this falls out for free: CI * test reporters, regression baselines, and a promptfoo bridge all consume it. */ export declare function checkReportToJUnit(report: CheckReport, opts?: { min?: number; per?: Readonly>; name?: string; }): string; /** Parse per-run cost/latency/tokens from a stream — pure, model-free. */ export declare function parseUsage(stdout: string): EvalUsage; /** Parse Claude Code's stream-json stdout into the common trace fields. */ export declare function parseClaudeRun(out: RunOut): ParsedModelRun; /** * The set of skills that RESOLVED (the `Skill` tool fired without error) in a run, * by their namespaced id (e.g. `superpowers:test-driven-development`). The * multi-skill generalization of {@link skillResolved}: where a trigger-rate run * asks "did skill X fire?", a SELECTION-collision run asks "which skills fired?" — * so a single pass over the plugin's prompts reveals whether one skill's prompt * wrongly activates a SIBLING (the behavioral confirmation of the deterministic * `description-overlap` proxy). Errored Skill calls are excluded, like * `skillResolved`. */ export declare function whichSkillsFired(trace: Trace): string[]; /** One selection-trial outcome: which skills fired (namespaced ids), or errored. */ export interface SelectionTrialResult { readonly fired: readonly string[]; readonly errored: boolean; } /** * Run ONE prompt against an installed plugin and report WHICH of its skills fired * — the per-run primitive behind the plugin selection-collision matrix * (`measurePluginSelection` in `scan-behavioral.ts`). Mirrors the trigger-rate * trial (throwaway cwd, fixture seeded, errored turn excluded) but returns the * fired-skill SET instead of a single boolean, so the whole N×N collision matrix * falls out of one pass over the prompts (N× cheaper than re-running per pair). */ export declare function runSkillSelectionTrial(args: { readonly prompt: string; readonly pluginDir: string; readonly runner: AgentRunner; readonly parse?: ModelOutputParser; readonly model: string; readonly effort?: string | number; readonly tools?: readonly string[]; readonly timeoutMs?: number; readonly fixture?: Record; readonly runError?: (out: RunOut) => string | null; }): Promise; /** Aggregate per-run metrics: mean for numbers, fraction-true (0..1) for booleans. */ export declare function aggregate(rows: readonly Metrics[]): Record; /** * Aggregate per-run metrics with spread: mean, sample std, standard error, and * n. The se/std let you judge whether an A/B gap between arms is real or noise — * a difference smaller than the combined se is not yet significant. */ export declare function aggregateStats(rows: readonly Metrics[]): Record; /** Aggregate per-run usage into an arm's cost / latency / token totals + means. */ export declare function aggregateUsage(usages: readonly EvalUsage[]): ArmUsage; /** * A model id is "dated" (honestly pinned) when it ends in an 8-digit date stamp, * e.g. `claude-haiku-4-5-20251001`. A floating alias (`haiku`, `sonnet`, or even * `claude-sonnet-4-6` with no date) can change underneath you — so a cached or * baselined result pinned to it can silently hide model drift. See * `research/eval-architecture.md` (honest model pinning). */ export declare function isDatedModel(model: string): boolean; /** * Capability tier of a model by FAMILY: haiku=1 < sonnet=2 < opus=3 (version is * ignored, so `claude-sonnet-4-6` and a dated Sonnet rank equal). An unrecognized * family returns `null` — unrankable, so the floor never blocks a model we can't * judge (fail-open on ranking). Used by the model floor; aliases and full/dated * ids both work. */ export declare function modelTier(id: string): number | null; /** * Is `model` a weaker tier than `floor`? Both must be rankable (see * {@link modelTier}); an unrankable model/floor is never "below" (fail-open). */ export declare function belowModelFloor(model: string, floor: string): boolean; /** * Build an **ephemeral run environment** for a model-driven run: a NEW env object * with a *fresh* `HOME` (and `TMPDIR`) pointed at the throwaway `opts.home`, the * OS essentials (`PATH`, `LANG`, `LC_*`, `TERM`), the harness's OWN auth, and * nothing else. Pure — no fs, no spawn. * * The rationale is "fresh HOME + only the harness credential injected, **not** a * blanket wipe": running a model-driven skill/agent is itself a side effect (the * *model*, not the author, chose the actions), so it should not be able to read * the real `~/.gitconfig` / `~/.ssh` / `~/.aws` or write to the real `~`. But the * real CLI must still AUTHENTICATE, so the auth names the harness declares * (`runtime.runEnv.keep`) are re-injected, and the parent session's identity * (`runtime.runEnv.sessionIdentity`) never is. Which names those are is the * adapter's fact, not this file's: see `RunEnvPolicy` in `core/runtime.ts`. * Because this needs no kernel features, it is the cross-platform * STATE-protection floor (lands on macOS immediately), orthogonal to the Linux * bubblewrap HOST confinement in `src/sandbox.ts`. * * Notably it does NOT pass through `GIT_*`, `GH_TOKEN`, `SSH_*`, `AWS_*` access * keys, a config-dir override, or any other undeclared variable — those are * exactly what an ephemeral run must not see. A too-narrow `keep` breaks auth * loudly (the run fails to start); a too-broad one leaks silently, which is why * it is an allowlist and not a prefix. * * @param base the source environment to filter (usually `process.env`). * @param opts.home the throwaway dir to set as `HOME`/`TMPDIR`. * @param opts.allow extra var NAMES to pass through (e.g. the `VIGILES_*` keys * the eval already injects). Layered ON TOP of the harness's `keep`; a * session-identity name is dropped even when listed here. */ export declare function ephemeralRunEnv(base: NodeJS.ProcessEnv | Record, opts: { home: string; allow?: readonly string[]; }): Record; /** * Seed the throwaway HOME with the harness's own auth FILE(s) — best-effort; * covers local file-based OAuth; the env-var/host-brokered path is covered by * `ephemeralRunEnv`. The files are the runtime's declared `keepHomeFiles`. * * COPIES (never symlinks) each path from `realHome` into `throwawayHome`, * creating parent dirs as needed; a symlink would let the model-driven run write * back to the real credential file, defeating ephemerality. A path that doesn't * exist in the real HOME is skipped silently (that user auths via env-var / host * broker instead). Pure fs — no env, no spawn. */ export declare function seedEphemeralHome(throwawayHome: string, realHome: string, keep?: readonly string[]): void; /** Whether a run's captured output looks like a rate-limit / overload. Pure. */ export declare function isRateLimited(out: RunOut): boolean; /** Map `worker` over `items` with at most `concurrency` in flight, order preserved. */ export declare function runPool(items: readonly T[], concurrency: number, worker: (item: T) => Promise): Promise; /** * The author-facing failure for a report whose stubs left a call unanswered, or * undefined. One owner for the rule — `vigiles eval`'s exit 2 and the lock's * refusal both read it — so a consumer whose `assert` calls no helper still * fails. Pure. */ export declare function unansweredInReport(report: Pick, stubs: readonly ToolStub[]): string | undefined; /** * `runEval` with its author-facing output, over any runner — so the real one and * a test's fake share one path. Besides the run it prints, to stderr, what the * caller must not miss: a skill arm that never activates, a stub call no rule * answered (the same text `vigiles eval` fails with — a script that calls * `runEval` itself, as the `withServices` examples do, gets the report back and * nothing else, so the library is not silent about it either), and what the * run spent. */ export declare function runEvalReporting(spec: EvalSpec, runner: AgentRunner): Promise; export declare function runEvalWith(input: EvalSpec, runner: AgentRunner): Promise; /** Format an eval report as a compact table for the console (mean ± se, pass^k). */ export declare function formatEvalReport(report: EvalReport): string; /** * Measure how reliably a skill/behaviour *triggers*. A skill's value is its * description firing on the right task — the #1 documented skill-authoring pain — * and that's a property of the real model, not the wiring (which the * deterministic tier already proves). Install the plugin natively (`pluginDir`), * give a set of varied `prompts`, and a `fired` predicate over the run's `Trace` * (reuse the bare predicates, e.g. `(t) => skillResolved(t, "x:y")`). */ export interface TriggerRateSpec { /** * A stable name for this trigger eval — required to engage the {@link lock} * (it keys the committed `.vigiles/eval-locks/.lock.json`). Two evals over * the same skill with different prompts get distinct names. Optional otherwise. */ readonly name?: string; /** * **The eval LOCK** — the CI staleness gate (see `src/eval-lock.ts`). With * `name` set, `vigiles eval --update` records this trigger-rate report locally * and `--check` verifies it against the current inputs (skill contents, prompts, * model) with NO model call. Mode normally comes from the CLI flags. */ readonly lock?: EvalLockOptions; /** * Plugin dir installed natively (`--plugin-dir`) so its skills/commands * activate. Provide this OR {@link skillsDir}, not both. */ readonly pluginDir?: string; /** * A directory of LOOSE skills (`//SKILL.md`, e.g. a repo's * `.claude/skills`) to trigger-test directly. vigiles packages them into a * throwaway `--plugin-dir` for you and removes it afterward — the one-liner * for repo-local skills that aren't a published plugin. Provide this OR * {@link pluginDir}, not both. */ readonly skillsDir?: string; /** The varied prompts to test the trigger against. */ readonly prompts: readonly string[]; /** * Optional *irrelevant* prompts the skill should **not** fire on — the * precision side of triggering. Firing on these is a false positive (a skill * whose description is too broad and hijacks unrelated work). When given, the * report adds {@link TriggerRateReport.falsePositiveRate} and * {@link TriggerRateReport.precision}; `prompts` alone measures recall only. */ readonly irrelevantPrompts?: readonly string[]; /** Did the behaviour fire on this run? e.g. `(t) => skillResolved(t, "x:y")`. */ readonly fired: (trace: Trace) => boolean; /** * Co-install these skill sources ALONGSIDE the skill-under-test so it competes * for selection as it does in the real harness — the **whole-harness** tier. * Each entry is a plugin dir (`skills/` or `.claude/skills/`) or a loose skills * dir (`/SKILL.md`); their skills are merged into one install under the * under-test plugin's name (the under-test skill wins a name collision, so its * `:` id still matches `fired`). * * WHY: skill selection is competitive and Claude Code evicts the least-used * skill descriptions under a context budget, so an ISOLATED trigger-rate (the * default, `installSet` absent/empty) **overstates recall and understates * false-positives**. Isolated is the cheap authoring loop; populate the set for * a release gate. See `research/isolated-vs-whole-harness-eval.md`. */ readonly installSet?: readonly string[]; /** * Replace each skill's BODY with a no-op stub (keeping its frontmatter — name + * description). Trigger-rate is decided by the frontmatter ALONE (the model * selects a skill before its body loads), so stubbing can't change what's * measured — it just stops the run executing an expensive procedure once the * skill fires. All descriptions stay present, so selection competition is * faithful. **Defaults to `true`** here: testing a description never needs the * body, so it's automated, not a knob you must remember. Set `false` only in the * rare case you want the real body to run. See {@link stubSkillBody}. */ readonly stubSkillBodies?: boolean; /** * Minimum number of prompts each set (relevant + irrelevant) must have. A * handful of prompts can't tell a real recall/precision rate from noise, so * the run is rejected before it spends a token. Default 10; lower it * deliberately for a genuinely narrow skill. */ readonly minPrompts?: number; /** * Reject the run when two prompts in a set are closer than this in NCD * (gzip-based distance, 0..1; 0 = identical) — near-duplicate prompts inflate * a rate without testing varied phrasings. Default 0.3 (the rule-dup threshold). */ readonly minDistance?: number; /** Trials per prompt. Default 1. */ readonly trials?: number; /** * Model alias/id. Default `"sonnet"` — the realistic selector most Claude Code * users run. NOT haiku: trigger-rate is a *selection* measurement and haiku is a * much weaker selector, so it under-reports recall (dogfooded: a skill scored * 0.50 on haiku vs 0.90 on Sonnet). Override for a cheaper-but-pessimistic run. */ readonly model?: string; /** * Reasoning budget for the run (`claude --effort`, e.g. `"low"` or an integer). * Lives here beside `model` because it is part of the MEASUREMENT — it moves the * output distribution, not the sample size — so it is hashed into the lock and * the cache, and never read from an env var. Omit for the harness default. */ readonly effort?: string | number; /** * Minimum model tier this eval may run on (haiku; /** * How many runs to execute in parallel across the whole prompts × trials grid. * Default 1 (serial, the politest to rate limits). Raise it to cut wall-clock on * a large prompt set or roster sweep — the `spacingSec` pause still applies per * run, so it stays best-effort polite. Mirrors `EvalSpec.concurrency`. */ readonly concurrency?: number; } /** Per-prompt trigger result: how many of its trials fired. */ export interface PromptTriggerStat { readonly prompt: string; readonly fired: number; readonly trials: number; /** `fired / trials` (0 when no trials). */ readonly rate: number; } export interface TriggerRateReport { /** Overall fraction of relevant runs in which the behaviour fired (recall, 0..1). */ readonly rate: number; /** Total relevant runs (prompts × trials). */ readonly n: number; readonly perPrompt: readonly PromptTriggerStat[]; /** * Fraction of *irrelevant* runs that wrongly fired (lower is better). Present * only when {@link TriggerRateSpec.irrelevantPrompts} was given. */ readonly falsePositiveRate?: number; /** * `relevantFired / (relevantFired + irrelevantFired)` — of all firings, the * share on the right prompts. Present only when irrelevant prompts were given * AND something fired (undefined when nothing fired at all). Pairs with `rate` * (recall) to catch a skill that fires on everything _or_ nothing. */ readonly precision?: number; /** Per-prompt stats for the irrelevant set. Present with irrelevant prompts. */ readonly perIrrelevant?: readonly PromptTriggerStat[]; /** * Competitor skills co-installed via {@link TriggerRateSpec.installSet}. `0` * (the default) means the skill was measured **ISOLATED** — so `rate` is an * UPPER bound on real recall and `falsePositiveRate` a LOWER bound, because * selection is competitive (a populated harness can evict or out-compete the * description). A non-zero count is the whole-harness measurement. */ readonly competitors: number; /** * The plugin namespace the skills actually installed under — the `` * half of the `:` id `skillResolved` matches. * * Reported because with `skillsDir` the name is chosen by the packager, not by * the caller, so the single most common cause of a 0% run was a value the * caller had no way to know. Optional so a report recorded before this field * still parses. */ readonly namespace?: string; /** * Runs EXCLUDED because the turn errored / was rate-limited (detected by the * driver's `runError`), present only when > 0. These are NOT counted in `n` or * as misses — so `rate` reflects only valid runs. A large `errored` relative to * `n` means the measurement is thin (e.g. a Codex usage limit was hit); re-run. */ readonly errored?: number; /** Cost / tokens SPENT across all runs (relevant + irrelevant) — feeds the cost summary. */ readonly usage: ArmUsage; /** * Present when the driver's harness measures trigger-rate on an EXPERIMENTAL * basis (copied from {@link EvalDriver.experimental}) — the number is not * validated and can be wrong. `formatTriggerRateReport` prints it as a loud * caveat. Absent = a supported, trustworthy measurement (Claude Code). */ readonly experimental?: string; } /** * The default (Claude Code) eval driver: real `claude` + stream-json parsing. * * This lives at the COMPOSITION ROOT (`src/eval.ts`) on purpose, not in * `src/adapters/claude-code/` — it is NOT a boundary leak. Claude Code is the * wired DEFAULT (`measureTriggerRate`/`runEval` fall back to it), so `eval.ts` * must reference it directly; wiring the default is precisely a composition * root's job. `codexEvalDriver` lives in its adapter dir instead because Codex * is caller-INJECTED (never a default), so `eval.ts` never imports it. Relocating * this into the adapter would make `eval.ts → adapters/claude-code → eval.ts` a * circular import (the shared `EvalDriver`/`ModelOutputParser` types live here). * The asymmetry reflects default-vs-injected, not a hexagonal violation. */ export declare const claudeEvalDriver: EvalDriver; /** * The Claude Code {@link HarnessLiveDriver} — the EXECUTING tiers' side of the * adapter, reached through `claudeCodeAdapter.liveDriver()`. * * It lives HERE, at the composition root, for exactly the reason * `claudeEvalDriver` above does: it is assembled from the wired default runner * and `whichSkillsFired`, and moving it into `src/adapters/claude-code/` would * make `eval.ts → adapters/claude-code → eval.ts` a cycle. The adapter reaches * it through a dynamic `import()`, so nothing pays for this graph until an * executing tier actually runs. */ export declare const claudeCodeLiveDriver: HarnessLiveDriver; /** * Package loose `//SKILL.md` skills into a throwaway plugin dir * that `claude --plugin-dir` accepts — so repo-local skills (e.g. `.claude/skills`) * can be trigger-tested without hand-rolling a `plugin.json`. Writes a minimal * `.claude-plugin/plugin.json` and copies each `/` (recursively, so * `references/` etc. come along) under `skills//`. Returns the temp plugin * dir; the caller removes it (`measureTriggerRate` does). Throws if the directory * is missing or holds no `/SKILL.md`. */ export declare function packageSkillsDir(skillsDir: string, opts?: { name?: string; stub?: boolean; }): string; /** * Rewrite a SKILL.md to keep its YAML frontmatter (the trigger surface — name + * description) but replace the body with a no-op stub. Trigger-rate is a property * of the frontmatter ONLY: the model picks a skill from its name + description * before the body is ever loaded, so the body is causally downstream of selection * and irrelevant to whether the skill fires. Stubbing it lets a trigger run stop * AT selection instead of executing an expensive multi-step procedure — cheaper, * faster, and side-effect-free, without changing what's measured. Pure. */ export declare function stubSkillBody(skillMd: string): string; /** * Build a throwaway plugin dir mirroring `pluginDir`'s skills with their BODIES * stripped (frontmatter kept) — the trigger surface a description/firing check * needs, without paying to run each skill's procedure. Keeps the original plugin * NAME so `:` ids still match. The caller removes the returned dir. * See {@link stubSkillBody} for why the body is irrelevant to selection. */ export declare function stubbedPluginDir(pluginDir: string): string; /** * Distance between two prompts in ~0..1 (0 = identical, higher = more * different) via Normalized Compression Distance over the normalized text. * Reuses {@link ncd} from the proof engine. */ export declare function promptDistance(a: string, b: string): number; export interface PromptDiversityIssue { readonly kind: "too-few" | "too-similar"; readonly message: string; } /** * Deterministically check a prompt set is big and varied enough to measure a * trigger rate: at least `minPrompts` entries, and no two closer than * `minDistance` in NCD. Pure — no model. `label` names the set in messages. */ export declare function checkPromptDiversity(prompts: readonly string[], opts?: { minPrompts?: number; minDistance?: number; label?: string; }): PromptDiversityIssue[]; /** Throw if a prompt set isn't big/varied enough. See {@link checkPromptDiversity}. */ export declare function assertPromptDiversity(prompts: readonly string[], opts?: { minPrompts?: number; minDistance?: number; label?: string; }): void; /** * Build a combined plugin: the under-test skills PLUS every `installSet` source's * skills, so the skill-under-test competes for selection as in the real harness. * Named after the under-test plugin so `:` ids still match; the * under-test skills win a name collision. Returns the dir + `added` = how many * installSet skills were merged in (excludes collisions). The report's * `competitors` is derived separately from the FULL pool (see `countSkills`), so * sibling skills already in the under-test source count too. Caller removes the * dir. Pure (filesystem only). */ export declare function packageInstallSet(opts: { underTestSrc: string; name: string; installSet: readonly string[]; stub: boolean; }): { dir: string; added: number; }; export declare function measureTriggerRateWith(spec: TriggerRateSpec, runner: AgentRunner, parse?: ModelOutputParser, runError?: (out: RunOut) => string | null, harness?: string): Promise; /** * Measure a skill/behaviour's real trigger rate across prompts × trials. Defaults * to the real `claude` CLI (`claudeEvalDriver`); pass `{ evalDriver }` to drive a * second harness — e.g. `measureTriggerRate(spec, { evalDriver: codexEvalDriver })` * from `vigiles/codex` (the eval-tier analog of `runHarnessTest`'s `{ adapter }`). * Requires that harness's binary + auth. */ export declare function measureTriggerRate(spec: TriggerRateSpec, opts?: { evalDriver?: EvalDriver; }): Promise; /** Format a trigger-rate report: overall %, then each prompt's rate. */ export declare function formatTriggerRateReport(report: TriggerRateReport): string; //# sourceMappingURL=eval.d.ts.map