/** * Evaluation Harness, EvalRunner. * * Runs eval suites using production runtime paths: * - PerfMonitor for budget evaluation * - SloCollector for SLO p95 measurements * - scoreScenario() for dimension scoring * * The runner exercises the production code paths that gate CI. */ import type { EvalScenario, EvalSuiteResult, EvalGateResult, EvalBaseline } from './types.js'; export interface EvalRunnerOptions { /** * Regression threshold for gate comparisons. * A scenario regresses if its composite score drops by more than this amount. * Default: 5 (5-point drop). */ regressionThreshold?: number | undefined; } /** * Runs eval suites and produces structured results. * * @example * ```ts * const runner = new EvalRunner(); * const result = await runner.runSuite(mySuite); * if (!result.passed) process.exit(1); * ``` */ export declare class EvalRunner { private readonly regressionThreshold; constructor(options?: EvalRunnerOptions); /** * Run all scenarios in a suite sequentially. * * Each scenario: * 1. Calls scenario.run() via the production code path * 2. Evaluates the raw result through PerfMonitor (if perfReport is absent) * 3. Scores the result via scoreScenario() * * @param suite - Suite name (used for logging and baseline matching). * @param scenarios - Scenarios to run. * @returns Aggregated EvalSuiteResult. */ runSuite(suite: string, scenarios: EvalScenario[]): Promise; /** * Compare a fresh suite result against a stored baseline and enforce the * absolute per-dimension floors. * * The gate fails when EITHER of these holds for any scenario: * 1. The scenario is below its absolute floor (`scorecard.passed === false`) * , checked for every fresh scenario, independently of the baseline. * 2. The scenario regressed more than `regressionThreshold` points versus * its baseline score. * * Scenarios present in the fresh run but absent from the baseline are NOT * silently skipped: they are still floor-checked (rule 1) and surfaced in * `unbaselined`. They simply cannot be regression-checked (rule 2) this run * because there is nothing to compare against, they seed the next baseline. * * @param fresh - Result from a freshly-run suite. * @param baseline - Previously stored baseline (may be undefined). * @returns Gate result with regression, floor-failure, and unbaselined entries. */ evaluateGate(fresh: EvalSuiteResult, baseline: EvalBaseline | undefined): EvalGateResult; private _runScenario; /** Execute a scenario's run() function, catching and normalising errors. */ private _executeScenario; } //# sourceMappingURL=runner.d.ts.map