/** * Runtime driver for `dql agent eval`. * * The in-process driver calls `answer()` directly, which skips * `createHybridRouter`, `AgentRunEngine`, `enforceOrdinaryAnalyticalPlanBoundary` * and `defaultAgentRunGates`. Every dead-end this harness exists to measure lives * in that skipped half — the router can block before the cascade runs, the engine * synthesizes its own modeling-gap block, and the gates declare refusal codes * terminal. Measuring the answer loop alone reports a cleaner picture than users * experience. * * This driver posts the same body the Ask panel posts and scores the persisted * `AgentRun`, so the suite exercises routing, gates, and the transport projection * as a real end-to-end contract. */ import type { AgentAnswer, AgentRun, AgentRunRoute, AgentRunStatus, AgentRunTrustState } from '@duckcodeailabs/dql-agent'; /** Eval-facing view of a run, shaped like the fields the scorer already reads. */ export interface RuntimeDrivenRun { kind: 'certified' | 'uncertified' | 'no_answer'; route?: 'certified' | 'generated_sql' | 'research' | 'clarify' | 'blocked'; runRoute: AgentRunRoute; status: AgentRunStatus; trustState: AgentRunTrustState; answer?: string; proposedSql?: string; rows?: unknown[]; clarificationOptionCount: number; refusalCode?: string; runId: string; /** * A conversational reply (greeting, capability, polite redirect). It asserts * nothing about the data, so it is neither an answer nor a refusal — scoring it * as either misreads the correct outcome for an out-of-scope question. */ conversational: boolean; /** * Did the meaning resolver actually run for this turn? * * With no provider configured it cannot, and `mayAssumeInterpretation` goes * false (AGT-017) — DQL deliberately refuses to settle `booked_revenue` vs * `billed_revenue` by lexical rank with semantic judgment switched off. Every * ambiguous question then clarifies. That is correct behaviour, but it makes a * clarification rate measured without a provider meaningless, so the harness * has to be able to say so. */ meaningResolved: boolean; /** * Count captured by the persisted router retrieval receipt. This is not a * synthetic context-pack size: runtime runs do not return an AgentAnswer * context pack through the transport projection. */ retrievalCandidateCount?: number; /** Source-lane coverage retained by the router-owned cascade receipt. */ sourceCoverage?: NonNullable['sourceCoverage']; /** Typed terminal authority, distinct from a user-facing clarification. */ terminalOutcome?: NonNullable['terminalOutcome']; /** Provider/tool count recorded by persisted runtime telemetry. */ toolCallCount: number; } /** * Did this run decline to produce a usable answer? * * `needs_review` is deliberately NOT a refusal: a review-required generated * answer is the intended outcome when no governed plan froze, and counting it as * a refusal would make the false-refusal metric punish the very behaviour the * cascade is supposed to produce. */ export declare function runtimeRunRefused(run: Pick): boolean; /** Map an engine route onto the vocabulary the eval cases already use. */ export declare function evalRouteForRun(route: AgentRunRoute): RuntimeDrivenRun['route']; /** * Collapse a run into the coarse `kind` the existing expectations assert. * Certification follows TRUST, not route: a certified route that degraded to a * review-required answer must not be scored as certified. */ export declare function evalKindForRun(run: Pick): RuntimeDrivenRun['kind']; /** Pull SQL and result rows out of the run's artifacts, when it produced any. */ export declare function runtimeRunOutputs(run: AgentRun): { proposedSql?: string; rows?: unknown[]; }; export declare function projectRuntimeRun(run: AgentRun): RuntimeDrivenRun; export interface RuntimeDriverOptions { runtimeBase: string; question: string; requestedMode?: 'ask' | 'auto' | 'research'; threadId?: string; timeoutMs?: number; fetchImpl?: typeof fetch; } /** * Post one question to a running `dql serve` and return the persisted run. * * Failures are surfaced rather than swallowed: a harness that silently scores a * transport error as a refusal would report a false-refusal spike that no code * change caused. */ export declare function driveViaRuntime(options: RuntimeDriverOptions): Promise; /** * Adapt a persisted run into the `AgentAnswer` shape the existing scorer reads, * so both drivers share one set of assertions. * * Fields the run genuinely does not carry are left UNDEFINED rather than * defaulted. An expectation that references one then fails loudly instead of * passing against a fabricated value — a harness that quietly invents * `sourceTier` would report agreement it never observed. */ export declare function answerFromRuntimeRun(run: AgentRun): AgentAnswer; //# sourceMappingURL=agent-eval-runtime.d.ts.map