/**
 * `dql eval` — Routing-accuracy harness for the DQL answer agent.
 *
 * Replays a golden set of questions through the EXISTING agent router
 * (`planAgentAnswer`) and scores how well DQL routes each question:
 * certified vs generated vs missing-context (refusal), which certified block
 * it selected, and whether the answer grain matches.
 *
 * The golden set comes from two sources:
 *   1. Every certified block's `examples[].question` in the compiled manifest.
 *      These expect route = certified and expect that same block to be selected.
 *   2. Optional `eval/*.yaml` files in the project, each a list (or { cases: [] })
 *      of `{ question, expectRoute, expectBlock?, expectGrain?, expectRefuse?, name? }`.
 *      These let a repo express the harder categories the block examples cannot:
 *      generated (Tier-2), insufficient-context (refusal), conflict, wrong-grain.
 *
 * This command READS the router output only. It never executes SQL against a
 * warehouse and never changes routing, the manifest schema, or the DQL language.
 *
 * Usage:
 *   dql eval [path]                          Replay the golden set, print a report
 *   dql eval [path] --format json            Emit a machine-readable JSON report
 *   dql eval [path] --min-route-accuracy 0.9 Fail (exit 1) below this route accuracy
 *   dql eval [path] --min-refusal 0.8        Fail (exit 1) below this refusal recall
 *   dql eval [path] --min-answer-rate 0.9    Fail below answer rate on answerable cases
 *   dql eval [path] --no-examples            Skip manifest block examples (yaml only)
 */
import { type DQLManifest } from '@duckcodeailabs/dql-core';
import { type PlanAgentAnswerResult } from '@duckcodeailabs/dql-agent';
import type { CLIFlags } from '../args.js';
/** Spec-facing route labels. The router speaks certified/generated_sql/research/clarify. */
export type EvalRoute = 'certified' | 'generated' | 'missing_context' | 'research';
/** The categories the harness can express and score. */
export type EvalCategory = 'certified' | 'generated' | 'insufficient_context' | 'conflict' | 'wrong_grain' | 'question_plan' | 'follow_up';
export interface EvalCase {
    /** Stable label for the report. Defaults to the question. */
    name: string;
    question: string;
    /** Where this case came from. */
    source: 'block_example' | 'yaml';
    /** Which scoring category this case exercises. */
    category: EvalCategory;
    /**
     * Prior turn replayed before this question. Its context pack is handed to the
     * follow-up (priorContextPackId + topic relation), so thread bugs — sticky
     * metric carry, poisoned working state, member-binding follow-ups — are
     * expressible as golden cases.
     */
    priorQuestion?: string;
    /** How this question relates to the prior turn. Default: continuation. */
    topicRelation?: 'continuation' | 'refinement' | 'return' | 'shift';
    /** Measures the prior turn answered with (models sticky-metric carry). */
    priorMeasures?: string[];
    expectRoute?: EvalRoute;
    /** Block name we expect the router to select (only meaningful for certified). */
    expectBlock?: string;
    /** Grain we expect the selected block to carry. */
    expectGrain?: string;
    /** When true, we expect a safe refusal (route = missing_context). */
    expectRefuse?: boolean;
    /** Terms that must appear among the question plan's metric terms (case-insensitive contains). */
    expectMetricTerms?: string[];
    /** Terms that must NOT appear among the metric terms (sticky-carry guard). */
    expectNoMetricTerms?: string[];
    /** Phrases that must survive as member filters in the requested shape. */
    expectFilters?: string[];
    /** Phrases that must NOT appear as member filters (governed-name misparse guard). */
    expectNoFilters?: string[];
    /** Object keys that must be present in the retrieved context pack. */
    expectEvidence?: string[];
}
export interface EvalCaseResult {
    name: string;
    question: string;
    source: EvalCase['source'];
    category: EvalCategory;
    passed: boolean;
    /** What the router actually did. */
    actualRoute: EvalRoute;
    actualBlock?: string;
    actualGrain?: string;
    hasBlockingMissingContext: boolean;
    /** Per-check expectations + outcomes for the human report and JSON. */
    expectRoute?: EvalRoute;
    expectBlock?: string;
    expectGrain?: string;
    expectRefuse?: boolean;
    routeMatch?: boolean;
    blockMatch?: boolean;
    grainMatch?: boolean;
    refusalMatch?: boolean;
    /** Question-plan shape outcomes (metric terms + member filters + evidence). */
    planShapeMatch?: boolean;
    /**
     * Was the expected block RETRIEVED at all, regardless of whether the router
     * then chose it? Separates a retrieval miss from a routing miss — without it
     * a drop in `blockSelectionAccuracy` cannot tell you which half broke.
     */
    retrievedExpectedBlock?: boolean;
    failures: string[];
    trace: EvalTraceStage[];
}
export interface EvalTraceStage {
    stage: 'context' | 'route' | 'scoring';
    status: 'passed' | 'failed' | 'not_run' | 'info';
    message: string;
    payload?: unknown;
}
export interface EvalScores {
    total: number;
    passed: number;
    /** answer rate: non-refusal route / cases that did not expect a safe refusal. */
    answerRate: number | null;
    /** route accuracy: cases whose expected route matched / cases with a route expectation. */
    routeAccuracy: number | null;
    /** block-selection accuracy: correct block / cases that expected a specific block. */
    blockSelectionAccuracy: number | null;
    /** grain-match precision: correct grain / cases that expected a specific grain. */
    grainMatchPrecision: number | null;
    /**
     * executable candidate recall: expected block present in the retrieved
     * context / cases that expected a specific block. The router can only choose
     * from what retrieval surfaced, so this bounds every selection metric above.
     */
    executableCandidateRecall: number | null;
    /** refusal precision: true refusals / all router refusals across the set. */
    refusalPrecision: number | null;
    /** refusal recall: refusals correctly produced / cases that expected a refusal. */
    refusalRecall: number | null;
}
export type EvalRouteDistribution = Record<EvalRoute, number>;
export type EvalCategoryDistribution = Record<EvalCategory, number>;
export type EvalSourceDistribution = Record<EvalCase['source'], number>;
export interface EvalDistributions {
    /** Router-selected cascade tier for each case. This is the PR drift signal. */
    actualRoutes: EvalRouteDistribution;
    /** Expected tier coverage from the golden set, when authored. */
    expectedRoutes: EvalRouteDistribution;
    categories: EvalCategoryDistribution;
    sources: EvalSourceDistribution;
}
export interface EvalReport {
    ok: boolean;
    scores: EvalScores;
    distributions: EvalDistributions;
    thresholds: {
        minRouteAccuracy: number | null;
        minRefusal: number | null;
        minAnswerRate: number | null;
        minCandidateRecall: number | null;
    };
    results: EvalCaseResult[];
}
/** Map the router's route vocabulary onto the spec's route labels. */
export declare function mapRoute(route: PlanAgentAnswerResult['routeDecision']['route']): EvalRoute;
/**
 * Build the golden set from a compiled manifest plus optional eval/*.yaml.
 * Each certified block contributes one case per example question.
 */
export declare function collectEvalCases(manifest: DQLManifest, yamlCases: EvalCase[], includeBlockExamples: boolean): EvalCase[];
/** Score a single case against the router's decision. Pure + unit-testable. */
export declare function scoreCase(testCase: EvalCase, plan: PlanAgentAnswerResult, manifest: DQLManifest): EvalCaseResult;
/** Aggregate per-case results into the spec's score set. Pure + unit-testable. */
export declare function computeScores(results: EvalCaseResult[]): EvalScores;
/** Aggregate stable PR-facing distribution counters from scored cases. */
export declare function computeDistributions(results: EvalCaseResult[]): EvalDistributions;
/** Load and normalize all `eval/*.yaml` cases for a project. */
export declare function loadYamlEvalCases(projectRoot: string): EvalCase[];
/**
 * Run the full eval: build (or read) the manifest, collect cases, replay each
 * through the router, score, and assemble the report. Side-effect free except
 * for invoking the (read-only) router. Returns a structured report.
 */
export declare function runEvalHarness(projectRoot: string, options: {
    includeBlockExamples: boolean;
    minRouteAccuracy: number | null;
    minRefusal: number | null;
    minAnswerRate?: number | null;
    minCandidateRecall?: number | null;
}): Promise<EvalReport>;
/** A threshold gate only fails when a configured threshold has measurable data below it. */
export declare function meetsThresholds(scores: EvalScores, minRouteAccuracy: number | null, minRefusal: number | null, minAnswerRate?: number | null, minCandidateRecall?: number | null): boolean;
export declare function runEval(pathArg: string | null, rest: string[], flags: CLIFlags): Promise<void>;
//# sourceMappingURL=eval.d.ts.map