/** * `vigiles audit` model trigger tier — the BEHAVIORAL column of the audit report. * * Structural `scan`/`scanPlugin` is deterministic, no-model, CI-free — and stays * that way. This is the model-gated column that stacks on top: for each * model-invocable skill in a plugin, it measures how reliably the description * actually FIRES (recall, + precision when irrelevant prompts are supplied), * reusing `measureTriggerRate`. It degrades honestly when the `claude` CLI / auth * is absent rather than faking a pass — exactly like the egress column. * * Prompts are AUTHOR-SUPPLIED (a per-skill JSON map), not model-generated — a * path in prose is undecidable, and the deterministic-input discipline is what * makes the column trustworthy. See `research/plugin-behavioral-findings.md`. */ import type { PluginLayout } from "./core/layout.js"; import type { HarnessDialect } from "./core/dialect.js"; import { type EvalDriver } from "./eval.js"; import { type EventFiringDriver, type HarnessLiveDriver } from "./core/live-driver.js"; import type { HarnessAdapter } from "./core/adapter.js"; /** Author-supplied prompt sets for one skill (bare skill name → these). */ export interface SkillPrompts { readonly prompts: readonly string[]; readonly irrelevant?: readonly string[]; } /** The `--prompts ` shape: bare skill name → its prompt sets. */ export type TriggerPromptSet = Record; export interface SkillTriggerResult { readonly skill: string; /** Whether a model probe actually ran (false = skipped, see `note`). */ readonly measured: boolean; readonly recall?: number; readonly precision?: number; readonly falsePositiveRate?: number; readonly n?: number; /** Why it was skipped, or a measurement error. */ readonly note?: string; } export interface BehavioralReport { /** False when the `claude` CLI / auth is absent — the column couldn't run. */ readonly available: boolean; readonly results: readonly SkillTriggerResult[]; /** * Set when the driving harness measures trigger-rate on an EXPERIMENTAL basis * (copied from {@link EvalDriver.experimental}) — Codex, whose firing is * inferred from a SKILL.md read and can be wrong. {@link formatBehavioralReport} * prints it as a loud caveat above the numbers. Absent = supported (Claude Code). */ readonly experimental?: string; } export interface ProbeOptions { readonly concurrency?: number; readonly model?: string; readonly minPrompts?: number; readonly minDistance?: number; /** Which harness drives it — the adapter itself, not its name (default * {@link defaultAdapter}). A reference-only adapter reports `available: false`. */ readonly adapter?: HarnessAdapter; /** Layout + dialect for candidate discovery — so a Codex repo's skills (under * the Codex layout) are found, not silently missed by the default CC layout. */ readonly layout?: PluginLayout; readonly dialect?: HarnessDialect; } /** The injectable core (for tests): probe every model-invocable skill that has prompts. */ export declare function probePluginTriggersWith(dir: string, promptSet: TriggerPromptSet, live: HarnessLiveDriver, opts?: ProbeOptions): Promise; /** * Probe a plugin's skills against the real harness (default Claude Code; Codex via * `opts.harness`). Needs that harness's binary + auth; degrades to * `available: false` otherwise. */ export declare function probePluginTriggers(dir: string, promptSet: TriggerPromptSet, opts?: ProbeOptions): Promise; /** Format the behavioral column as a scan-report section. */ export declare function formatBehavioralReport(b: BehavioralReport): string; /** Knobs for the selection-collision measurement. */ export interface SelectionOptions { /** Repeats per prompt (default 1). */ readonly trials?: number; /** Selector model — defaults to Sonnet (a weaker model under-selects). */ readonly model?: string; /** * Reasoning budget (`claude --effort`) for the selector, or undefined for the * harness default. Present for {@link measureSelectionMatrix}, the ASSERTABLE * test primitive, where the configuration a number came from has to be pinnable. * * Deliberately NOT exposed as an `audit` CLI flag: `audit` is a local report, * not a reproducibility surface, and a flag nobody can act on is surface without * a use. The audit probe therefore leaves this unset and runs at the default. */ readonly effort?: string | number; /** Parallel runs across the prompts × trials grid (default 1). */ readonly concurrency?: number; /** Which harness drives it — the adapter itself (default {@link defaultAdapter}). * A harness whose firing signal is INFERRED rather than a discrete event reports * n/a: the matrix asks WHICH skill fired, which an inference cannot answer. */ readonly adapter?: HarnessAdapter; /** * @deprecated Pass {@link SelectionOptions.adapter} instead. Kept for one * release because this options type is public API (`vigiles/claude-code`); * a name is resolved through the registry, never compared to a literal. */ readonly harness?: string; /** Layout + dialect for candidate discovery, mirroring {@link ProbeOptions} — * so a Codex repo's skills are found under the Codex layout, not silently * missed by the Claude Code one. */ readonly layout?: PluginLayout; readonly dialect?: HarnessDialect; } /** One run's outcome for the matrix: which of the plugin's OWN skills fired. */ interface SelectionRun { readonly intended: string; readonly firedBare: readonly string[]; } export interface SkillSelectionStat { readonly skill: string; /** Fraction of its own prompts on which it fired (matrix diagonal). */ readonly recall: number; /** Fraction of its own prompts on which a SIBLING skill also/instead fired. */ readonly collisionRate: number; /** Non-errored runs measured for this skill. */ readonly n: number; /** Sibling skills that fired on this skill's prompts, by rate (desc, rate>0). */ readonly collidesWith: readonly { readonly skill: string; readonly rate: number; }[]; } export interface SelectionReport { /** False when the harness CLI / auth is absent, or the harness has no selector. */ readonly available: boolean; /** Matrix axes — the plugin's model-invocable skill names (bare). */ readonly skills: readonly string[]; /** matrix[i][j] = times skill j fired when skill i's prompt was given. */ readonly matrix: readonly (readonly number[])[]; readonly perSkill: readonly SkillSelectionStat[]; /** Plugin-level: fraction of all runs where a non-intended skill fired. */ readonly collisionRate: number; readonly n: number; readonly note?: string; } /** * Pure aggregation: fold per-run fired-skill sets into the N×N selection matrix + * per-skill recall/collision + the plugin-level collision rate. Separated from the * model-driving so it's unit-testable with synthetic runs (no model). */ export declare function buildSelectionReport(skills: readonly string[], runs: readonly SelectionRun[]): SelectionReport; /** * The injectable core (for tests): drive the matrix via a fake/real driver. * * 🔴 IT TAKES AN {@link EventFiringDriver}, NOT ANY LIVE DRIVER, AND THAT IS THE * RATCHET. The matrix asks WHICH of the plugin's skills fired on each prompt; a * harness that only INFERS firing cannot answer that, and the old guard was a * name check in the wrapper below — invisible to anyone calling this core * directly. Now an inferred driver is a compile error at every call site. */ export declare function measurePluginSelectionWith(dir: string, promptSet: TriggerPromptSet, probe: EventFiringDriver, opts?: SelectionOptions): Promise; /** * Measure a plugin's cross-skill selection-collision matrix against the real * harness (Claude Code only — Codex has no skill-selection event). Needs the * `claude` CLI + model auth; degrades to `available: false` otherwise. */ export declare function measurePluginSelection(dir: string, promptSet: TriggerPromptSet, opts?: SelectionOptions): Promise; /** Format the selection-collision matrix as a scan-report section. */ export declare function formatSelectionReport(r: SelectionReport): string; /** Options for {@link measureSelectionMatrix}: {@link SelectionOptions} plus an * optional explicit prompt set (auto-derived from descriptions when omitted). */ export interface SelectionMatrixOptions extends SelectionOptions { /** * Per-skill recall prompts. Omit for ZERO-SETUP — prompts are auto-derived * from each skill's description (the same generator the audit trigger tier * uses). Supply your own for a curated collision benchmark; only the `prompts` * array per skill is read (any `irrelevant` bank is ignored here). */ readonly prompts?: TriggerPromptSet; } /** * Measure a plugin's skill-SELECTION collision matrix — "when I ask for skill i's * job, does ONLY skill i fire?" The first-class, assertable form of the cross-skill * collision measurement (pair with {@link assertNoCollision}). The matrix diagonal * is recall; off-diagonal mass is collision — skill j hijacking skill i's prompt, * the failure that breaks a multi-skill plugin and that per-skill trigger-rate * (each skill in ISOLATION) structurally can't see. * * ZERO-SETUP: with no `prompts`, they're derived from each model-invocable skill's * description. Claude Code only (Codex has no skill-selection event to read); needs * the `claude` CLI + model auth, else `available: false`. Thin promotion of * {@link measurePluginSelection}. See research/plugin-selection-collision.md. */ export declare function measureSelectionMatrix(dir: string, opts?: SelectionMatrixOptions): Promise; /** * Injectable core of {@link measureSelectionMatrix} (for tests): auto-derive the * prompts (unless supplied) and drive the matrix via a fake/real probe. */ export declare function measureSelectionMatrixWith(dir: string, probe: EventFiringDriver, opts?: SelectionMatrixOptions): Promise; /** * Assert a plugin's skills don't hijack each other — the gate over a * {@link SelectionReport} from {@link measureSelectionMatrix}. `maxOffDiagonal` * caps EACH skill's collision rate (fraction of its own prompts on which a SIBLING * fired); `maxPluginCollision` caps the plugin-wide rate. With neither set it * demands ZERO collision. THROWS (never a silent green) when nothing was measured * — an unavailable harness or a zero-run report is a gap, not a pass. */ export declare function assertNoCollision(report: SelectionReport, opts?: { maxOffDiagonal?: number; maxPluginCollision?: number; }): void; /** Does a skill description assert a hard constraint (→ an adversarial-gate candidate)? */ export declare function isGateDescription(description: string): boolean; /** A skill considered for gate detection — name + its (model-visible) description. */ export interface GateCandidate { readonly name: string; readonly description?: string; readonly userInvoked?: boolean; readonly hasDescription?: boolean; } /** * The model-invocable, described skills whose description reads as an enforcement * gate — the candidates for the adversarial-gate eval. User-invoked and * description-less skills are excluded (they can't auto-fire a constraint on the * model's behaviour), mirroring the trigger-rate candidate filter. */ export declare function detectGateSkills(skills: readonly GateCandidate[]): readonly string[]; /** A gate under test: its name + the rule its description states. */ export interface GateUnderTest { readonly name: string; readonly description: string; } /** The verdict an injected judge returns (a subset of judge.ts JudgeResult). */ export interface GateVerdict { readonly pass: boolean; readonly score: number; readonly reason: string; } /** Injected dependencies, so the orchestration is unit-testable with no model. */ export interface GateEvalDeps { readonly driver: EvalDriver; /** Grade whether the gate held, given the run output + the rule rubric. */ readonly judge: (a: { output: string; rubric: string; }) => GateVerdict; /** Turn a gate's rule into a one-line user request that tries to violate it. */ readonly derive: (gate: GateUnderTest) => string; } export interface GateOptions { /** Selector model for the harness run (default sonnet). */ readonly model?: string; /** Attacks per gate (default 1). */ readonly trials?: number; /** Concurrent harness runs (default 1). */ readonly concurrency?: number; /** Which harness drives it — the adapter itself (default {@link defaultAdapter}). */ readonly adapter?: HarnessAdapter; readonly layout?: PluginLayout; readonly dialect?: HarnessDialect; /** Author-supplied attack prompts (bare skill name → prompts); overrides derive. */ readonly attacks?: Record; } export interface GateResult { readonly skill: string; readonly measured: boolean; /** Fraction of attacks the gate HELD (1 = held every time). */ readonly heldRate?: number; /** Convenience: held on EVERY attack (a single cave → false). */ readonly held?: boolean; readonly n?: number; /** The attack used (first), for the report. */ readonly attack?: string; /** The judge's rationale on a representative cave (else a hold), for the report. */ readonly reason?: string; readonly note?: string; } export interface GateAdversarialReport { readonly available: boolean; readonly results: readonly GateResult[]; readonly note?: string; } /** Build the LLM-judge rubric from the gate's own rule (pure). */ export declare function gateRubric(gate: GateUnderTest): string; /** The injectable core (unit-testable with fake driver/judge/derive — no model). */ export declare function measureGateAdversarialWith(dir: string, gates: readonly GateUnderTest[], deps: GateEvalDeps, opts?: GateOptions): Promise; /** * Measure whether a plugin's enforcement-gate skills HOLD when adversarially * challenged. Detects gate skills (keyword heuristic), auto-derives an attack from * each rule (unless author-supplied), runs the UNSTUBBED harness, and LLM-judges * hold vs cave. Claude Code only; degrades to `available: false` without the CLI/auth. */ export declare function measureGateAdversarial(dir: string, opts?: GateOptions): Promise; /** Default adversarial attacks per gate (stochastic → need >1; unstubbed → keep low). */ export declare const DEFAULT_GATE_TRIALS = 3; /** Format the adversarial-gate report as a scan-report section. */ export declare function formatGateReport(r: GateAdversarialReport): string; export {}; //# sourceMappingURL=scan-behavioral.d.ts.map