/** * Red-Team Resistance Harness. * * The accuracy benchmark (`run-benchmark`, `run-llm-benchmark`) measures * how well the engine *detects* vulnerabilities in code. This harness * measures the complementary thing the roadmap calls for: a reproducible * **behavioral resistance score** for an agent / MCP target — how well it * survives a prompt-injection battery, plus the tool-scope / exfiltration * surface it exposes. * * It is a scoring layer over primitives that already exist * (`runAgentScanners` → the prompt-injection fuzzer, exfil-path graph, * sandbox audit, credential-scope audit, manifest audit). Those scanners * are deterministic and the payload corpus is fixed, so the score is * reproducible run-to-run for a given manifest — a number you can put on a * slide and a gate you can fail a build on. * * Two deliberately separate measures, because they behave differently: * * - **Injection resistance (headline):** the share of tools that resisted * every payload in the battery. A behavioral, normalized 0–100 score. * - **Exposure surface (context):** counts of exfil paths / sandbox * escapes / over-scoped credentials / manifest issues. These scale with * tool count and breadth — a broad security tool legitimately has a * large surface (which is exactly why our own self-cert *exempts* the * sandbox/exfil scanners as false-positive-heavy for this codebase). So * they are reported as surface, NOT folded into the resistance score * where raw counts would meaninglessly floor it to zero. * * Honest scope: this is the reproducible, offline/static floor (tool * surface, manifest hygiene, susceptibility patterns), not a live agent * executing payloads against a running model. A live-runtime battery is a * future extension. * * @module eval/redteam-harness */ import type { AgentScannerType, MCPManifest } from "../scanners/agent/types.js"; import type { DeterministicFinding } from "../scanners/types.js"; import type { Severity } from "../certification/types.js"; /** Exposure dimensions and the scanner that feeds each. */ export declare const EXPOSURE_DIMENSIONS: ReadonlyArray<{ dimension: string; scanner: AgentScannerType; }>; export interface InjectionResistance { /** 0–100: share of tools that resisted every payload in the battery. */ score: number; grade: string; toolsTested: number; vulnerableTools: number; resistantTools: number; } export interface ExposureDimension { dimension: string; scanner: AgentScannerType; findingCount: number; /** critical + high findings — the ones worth triage. */ criticalHigh: number; bySeverity: Record; clean: boolean; } export interface RedTeamReport { target: string; manifestName: string; manifestVersion?: string; corpus: "quick" | "standard" | "thorough"; injectionResistance: InjectionResistance; exposure: ExposureDimension[]; /** Headline resistance score (== injectionResistance.score). */ overallScore: number; grade: string; } /** * Tool-level injection resistance from the fuzzer's findings: a tool is * "vulnerable" if it has any per-tool prompt-injection finding (the * `prompt-injection:summary` aggregate is excluded). Resistance is the * share of tested tools with no such finding. Pure and deterministic. */ export declare function injectionResistanceScore(findings: DeterministicFinding[], toolsTested: number): InjectionResistance; /** Summarize one exposure dimension's findings (counts, not a 0–100 score). */ export declare function summarizeExposure(dimension: string, scanner: AgentScannerType, findings: DeterministicFinding[]): ExposureDimension; export interface RedTeamOptions { manifest: MCPManifest; target?: string; corpus?: "quick" | "standard" | "thorough"; /** Source path for the source-reading scanners (sandbox audit). */ sourcePath?: string; } /** * Run the red-team battery against a manifest and return a reproducible * resistance report. Runs only the behavioral / red-team-relevant agent * scanners (no supply-chain network calls, no baseline-dependent drift). */ export declare function runRedTeamBenchmark(options: RedTeamOptions): Promise; //# sourceMappingURL=redteam-harness.d.ts.map