import type { HarnessAdapter } from "./core/adapter.js"; import type { HarnessTestDriver, ToolCall, HookFire, ParsedRun, ModelTurn, ModelRequest } from "./core/harness-driver.js"; import type { Trace, SubagentTrace } from "./core/eval-driver.js"; export type { Trace, SubagentTrace } from "./core/eval-driver.js"; import { parseReplies, reachedModel } from "./adapters/claude-code/replies.js"; /** Re-exported for the eval tier's parser, which reads the same stream. */ export { parseReplies, reachedModel }; import type { HarnessRuntime } from "./core/runtime.js"; import { type SandboxMode } from "./sandbox.js"; export { scriptModel } from "./mock-model.js"; export type { ModelTurn, ModelRequest, ToolCall, HookFire, HarnessTestDriver, } from "./core/harness-driver.js"; import { loadPlugin, resolveHarness } from "./adapters/claude-code/plugin-loader.js"; export { loadPlugin, resolveHarness }; export { decideSandbox, specTrusted, sandboxAvailable, type SandboxMode, } from "./sandbox.js"; export interface HarnessTestSpec { /** Fixture files to write in a fresh temp working dir (path → contents). */ readonly files?: Record; /** `.claude/settings.json` contents — the hooks/permissions under test. */ readonly settings?: unknown; /** * Path to a real plugin/repo whose harness (hooks + CLAUDE.md + skills) is * loaded into the sandbox, so you test the assembled machine, not a retyped * subset. Inline `settings`/`files` layer on top. See src/plugin-loader.ts. */ readonly plugin?: string; /** * Path to a plugin dir to install NATIVELY via `claude --plugin-dir`, so its * skills / commands / agents / hooks register and ACTIVATE the real way — a * scripted `Skill` tool_use resolves, and the real model can trigger them. * Unlike `plugin` (which materializes a file subset that does NOT register * skills for the `Skill` tool), this is the real install path, so point it at a * COMPLETE plugin (internal references resolve). Inline `settings`/`files` and * `plugin` still layer on top. Resolved to an absolute path. */ readonly pluginDir?: string; /** * Path to an OUTPUT STYLE file to switch on for this run. The style is * written where the harness keeps styles, and selected by the name the * harness reads from the file — never by a name you type, because a setting * that misses the name loads no style and raises no error. * * The run is then checked: if no model request carried the style, * `runHarnessTest` THROWS instead of returning a trace, so a style that never * loaded cannot pass a test. Refused up front on a harness without output * styles, or when the fixture's files or settings already decide. * * What it proves is delivery. A scripted model does not read the style, so * nothing in this tier shows that a model FOLLOWS it. */ readonly outputStyle?: string; /** The scripted model turns the agent will take. */ readonly model: readonly ModelTurn[]; /** The user prompt. Default: "go". */ readonly prompt?: string; /** * Which tools EXIST in the session (`claude --tools`, Claude Code 2.0.31+). A * tool left out is not offered at all, so a call to it is "No such tool * available". Omitted, every tool exists. This is the field that withholds a * tool. Names only: a permission rule such as `Bash(git *)` is refused (put the * rule in `allowedTools`). `[]` is refused (an agent with no tools is never * served a scripted turn), and a scripted call to a tool the list leaves out * throws before the run starts. An MCP tool is not a built-in, so `--tools` * does not withhold it. Codex has no equivalent, so a run there refuses it. * * | field | Claude Code flag | meaning | * | -------------- | ----------------- | ---------------------------- | * | `tools` | `--tools` | which tools exist | * | `allowedTools` | `--allowedTools` | which are pre-approved | */ readonly tools?: readonly string[]; /** * Which tools are PRE-APPROVED (`claude --allowedTools`), so they do not stop * on a permission prompt. Takes permission rules (`Bash(git *)`). Omitted: * Read, Edit, Write and Bash are approved. An explicit list REPLACES that * default, so `tools: ["Read", "Write"]` with `allowedTools: ["Skill"]` leaves * Write unapproved. `[]`: nothing is approved. * * It never removes a tool, and leaving a tool out is not a fence: in headless * mode Claude Code refuses only the calls it would have asked about. For Bash * that is a command that creates, changes or removes a file (`touch`, `>`), * inside or outside the working directory; a command with no file effect * (`ls`, `printf`, `id`) runs unapproved. So a refused `touch` proves the * file-change guard is live, not that Bash is withheld. To withhold a tool use * `tools`; to forbid its use put a deny rule in the fixture's * `settings.permissions.deny`. */ readonly allowedTools?: readonly string[]; /** * Capture the full event transcript (`--output-format stream-json`) into * `stdout`, instead of just the final result object, so you can assert on what * the agent's tools returned — e.g. the body a `Skill` tool_use resolved. With * this on, `stdout` is newline-delimited JSON events, not a single object. */ readonly transcript?: boolean; /** Per-run wall-clock timeout in ms. Default 60000. */ readonly timeoutMs?: number; /** * Confinement policy for the code this run executes (`src/sandbox.ts`). * Default `"auto"` is safe-by-default: an inline-only spec (you authored it) * runs directly, but an external `plugin` / `pluginDir` brings in untrusted * third-party hooks and is run under bubblewrap — or, if no sandbox is * available, the run REFUSES rather than executing unconfined. Pass `false` to * opt out and run unconfined (you audited the code, or trust the outer * container); `"strict"` to force confinement even for trusted code. * * NOTE: confined execution is **Linux only** (bubblewrap is a Linux tool). On * macOS / Windows no sandbox is available, so an untrusted run will REFUSE * under `"auto"`/`"strict"` — use `sandbox: false` there if you trust the code. */ readonly sandbox?: SandboxMode; } export interface HarnessTestResult extends Trace { readonly exitCode: number; readonly stdout: string; /** Hook block messages and diagnostics land here. */ readonly stderr: string; /** The temp working dir (inspect or clean it up). */ readonly cwd: string; /** Number of model turns the agent took (mock turns served). */ readonly turns: number; /** Remove the temp working dir. */ cleanup(): void; } /** * Parse `--output-format stream-json` (the `transcript: true` output) into the * tools the agent invoked, each joined to its result by id. Returns [] for the * non-stream `json` output. The seam that lets a test assert on the agent's * actions, not a brittle stdout substring. */ export declare function parseToolCalls(streamJson: string): ToolCall[]; /** * Recover sub-agent runs as nested traces. A subagent-dispatch tool call (the * `Agent` tool on the live CLI — older docs say `Task` — carrying an * `input.subagent_type`) spawns a subagent whose own events the CLI tags with a * top-level `parent_tool_use_id` = the dispatch tool-use id. We group those * tagged tool calls under their dispatch, keyed by `subagent_type`. **Schema * verified against real claude output** (`parent_tool_use_id` sibling of * `message`, `subagent_type` in the dispatch input; tool named `Agent`) — the * same `message.content` line shape `parseToolCalls` consumes, and we match the * input field NOT the tool name so a future rename can't break it. Pure; empty * for a harness that doesn't emit `parent_tool_use_id` (e.g. Codex). */ export declare function parseSubagents(streamJson: string): SubagentTrace[]; /** * The terminal `result` event — present in BOTH `--output-format` shapes (a * `{type:"result", …}` line in stream-json, the single object in `json`), or * null. The seam for the final answer + turn count without parsing twice. */ export declare function parseResultEvent(stdout: string): Record | null; /** The agent's final answer text from a transcript / result object, or "". */ export declare function parseOutput(stdout: string): string; export declare function parseHooks(stdout: string): HookFire[]; /** * The names `--tools` takes for a `tools` list: trimmed and deduplicated. * `--allowedTools` takes permission rules (`Bash(git *)`); `--tools` takes tool * NAMES, and a rule there makes the CLI offer no tools at all. Reducing it to * `Bash` would offer more than the author wrote, so a rule is refused: it * belongs in `allowedTools`. Pure. */ export declare function toolAvailabilityList(tools: readonly string[]): readonly string[]; /** * A scripted call to a tool an EXPLICIT `tools` list leaves out. The CLI answers * it with "No such tool available" and the run goes on, so a test that scripts * the call and then asserts on something else passes over a step that never * happened. A tool that exists but is not in `allowedTools` is NOT this: the CLI * refuses it for permission, which is a result a test can assert on. Returns the * message to throw, or undefined. MCP tools are not built-ins, so `--tools` does * not withhold them and they are skipped. Pure. */ export declare function unofferedScriptedTool(model: readonly ModelTurn[], tools: readonly string[] | undefined): string | undefined; /** * The agent's own tool results, read off the requests it sent back to the model, * that say a scripted tool does not exist. The CLI answers an unknown name * (`Bsh` for `Bash`, an `mcp__` server that is not configured) with an error * RESULT and carries on, exit 0, so a test that asserts "the call errored and * left no side effect" passes on a typo. `tools` cannot catch it: the pre-flight * skips `mcp__` names and only runs when `tools` is given. Side-channel requests * are the CLI's own bookkeeping and are not read. Returns the message to throw, * or undefined. Pure. */ export declare function noSuchToolMessage(model: readonly ModelTurn[], requests: readonly ModelRequest[]): string | undefined; /** * A run on a `claude` that predates `--tools` exits at argument parsing, and * commander's "unknown option" says nothing about WHY vigiles passed it. Returns * the message to throw, or undefined when the run did not die that way. Pure. */ export declare function unsupportedToolsFlag(stderr: string): string | undefined; /** * The `claude` CLI argv for a harness run (shared by the direct and sandboxed * paths). `ANTHROPIC_BASE_URL` is set by the caller's environment / wrapper, not * here. Pure, so the arg shape is unit-tested. */ export declare function buildClaudeArgs(spec: HarnessTestSpec, hasSettings: boolean): string[]; /** Parse the `claude` stdout/stream into the unified trace fields. */ export declare function parseClaudeRun(stdout: string): ParsedRun; /** * Where a harness-tier run's agent gets HOME and the rest of its environment. * * - `inherit` — the caller's env (real HOME), minus the harness's declared * session identity. The default of `runHarnessTest`: a test that reads the * machine's config keeps doing so, but no run executes AS the caller's live * session. * - `throwaway` — a fresh HOME/TMPDIR at `dir`, the OS essentials, and nothing * else from the caller: no harness auth (the scripted mock needs none, and a * config-dir variable would point the run back at the real config), no * identity, no secrets. The same scrubbed shape as the eval tier's * `env: { kind: "ephemeral" }` (`scrubbedRunEnv`), without the auth. */ export type HarnessRunHome = { readonly home: "inherit"; } | { readonly home: "throwaway"; readonly dir: string; }; /** * The spawn env of one harness-tier run: {@link HarnessRunHome} applied to * `base`, with the runtime's mock wiring on top. Pure — the testable seam of * the un-coverable spawn. */ export declare function harnessSpawnEnv(runtime: HarnessRuntime, wiredEnv: Readonly>, home: HarnessRunHome, base?: NodeJS.ProcessEnv): Readonly>; /** Whether the agent CLI is available — harness tests need it. */ export declare function claudeAvailable(): boolean; /** * The Claude Code `HarnessTestDriver`: the existing argv/mock/parse seams bundled * behind the port the adapter-driven runner dispatches through. Behaviourally * identical to the previous hard-wired path. */ export declare const claudeCodeDriver: HarnessTestDriver; /** Options for {@link runHarnessTest}. */ export interface RunHarnessTestOptions { /** * Which harness to drive. Defaults to Claude Code. Pass `codexAdapter` * (`vigiles/codex`) to drive real `codex exec` against its Responses mock. * The adapter must support pillar 2 (`capabilities.harnessTesting`) and carry * a `harnessTestDriver`. */ readonly adapter?: HarnessAdapter; } /** Say so on stderr when side-channel calls arrived and no script turn was served. */ export declare function warnUnconsumed(count: number, sideChannelCount: number): void; /** * What makes a finished run unsound, most specific first, or undefined. Pure. */ export declare function runProblem(stderr: string, model: readonly ModelTurn[], modelRequests: readonly ModelRequest[], tools?: readonly string[]): string | undefined; /** * Run the real agent CLI against a scripted mock model, with the given fixture * and settings (hooks). Deterministic — same script, same result. Adapter-driven * (`opts.adapter`, default Claude Code): the Claude Code path is unchanged * (incl. the safe-by-default sandbox); pass `codexAdapter` to drive real codex. * * Safe by default (Claude Code): an external `plugin` / `pluginDir` brings in * untrusted third-party hooks and is confined under bubblewrap (`spec.sandbox`, * default `"auto"`); if no sandbox is available the run REFUSES rather than * executing unconfined. See `src/sandbox.ts`. The sandbox path is Claude Code * only — requesting confinement for another harness throws. */ export declare function runHarnessTest(spec: HarnessTestSpec, opts?: RunHarnessTestOptions): Promise; /** * {@link runHarnessTest} with the run's HOME chosen by the caller — see * {@link HarnessRunHome}. Not on a public entry point: the one caller that needs * a throwaway HOME is vigiles's own preflight (`output-style-arms.ts`), whose * run must not depend on, or act as, the machine it happens to run on. The * confined (bubblewrap) path ignores `home`: it already runs with its own HOME * and a cleared env. */ export declare function runHarnessTestIn(spec: HarnessTestSpec, opts: RunHarnessTestOptions, home: HarnessRunHome["home"]): Promise; /** * `runHarness` — the harness-scope entry of the revamped API (Phase 2 of * `research/testing-api-design.md`). The harness has two execution scopes, `hook` * (`runHook`) and `harness` (the whole assembled agent); today's `integration` / * `e2e` / `eval` are all the **harness** scope under realness flags. This entry is * the **deterministic** harness run (`model: "mock"`, the default) — the * workhorse you gate every commit, with no key. A **real-model** harness run is * non-deterministic by definition, so you don't *assert* a single one — you * `measure()` it across trials (the eval scope). `egress` is a capability of this * scope (the e2e tier), not a separate tier. * * Behaviour is identical to `runHarnessTest` (which it wraps); the new name + * `model` flag make the scope/realness explicit and steer real-model runs to the * right tool. */ export declare function runHarness(spec: HarnessTestSpec, opts?: RunHarnessTestOptions & { model?: "mock" | "real"; }): Promise; /** * The files and settings a run with `outputStyle` writes — the style at its * place, selected by the name the harness reads from it. The same fixture * `runHarnessTest({ outputStyle })` runs, so an eval arm built from it gets * exactly what that run proved reaches the model. */ export declare function outputStyleFixture(outputStyle: string, adapter?: HarnessAdapter): { readonly files: Readonly>; readonly settings: unknown; }; //# sourceMappingURL=harness-test.d.ts.map