import { b as Predicate, I as IterationTranscript, c as PredicateResult, A as ArgMatcher, d as TranscriptToolCall, a as TranscriptUsage, R as RenderObservationSummary, T as ToolErrorRecord } from '../types-BBk0lTBo.js'; export { e as ArgMatchMode, C as CasePredicates, f as PREDICATE_PLACEHOLDER_STRINGS, g as PredicatePlaceholder, P as PredicateScope, h as PredicateType, i as RenderObservationStatus, j as TURN_SCOPABLE_PREDICATE_KINDS, k as ToolErrorKind, l as argMatcherSchema, m as casePredicatesSchema, n as isTurnScopablePredicateKind, p as predicateArraySchema, o as predicateSchema, q as predicateScopeSchema } from '../types-BBk0lTBo.js'; import { E as EvalTraceInput } from '../index-fZyfLCHE.js'; export { e as extractToolErrors } from '../eval-tool-execution-CcInGlXP.js'; import 'zod'; import 'ai'; import '../public-types-CX8stXC3.js'; import '../types-HXAijHji.js'; import '../types-CI0Xyszt.js'; import '@modelcontextprotocol/client'; import '@modelcontextprotocol/client/stdio'; /** * Pure evaluator for the state-based predicate library. * * `evaluatePredicates(transcript, predicates)` returns one * {@link PredicateResult} per predicate; `allPredicatesPassed` reduces them to * the case verdict (a case passes iff **all** predicates pass; zero predicates * pass vacuously). Every function here is a pure function of its inputs — no * I/O, no clocks, no randomness — which is exactly what makes predicates a * valid CI gate. */ /** Evaluate a single predicate against the iteration transcript. */ declare function evaluatePredicate(transcript: IterationTranscript, predicate: Predicate): PredicateResult; /** Evaluate every predicate, preserving order. */ declare function evaluatePredicates(transcript: IterationTranscript, predicates: Predicate[] | undefined): PredicateResult[]; /** * Case verdict from predicate results: passes iff **all** pass. An empty set * passes vacuously (a case with no predicates is not gated by predicates). */ declare function allPredicatesPassed(results: PredicateResult[]): boolean; /** One prompt turn's checks plus the turn-scoped transcript to run them on. */ interface TurnChecksInput { /** Zero-based index of the turn in the case's `promptTurns`. */ promptIndex: number; /** The turn's per-turn checks (already restricted to turn-scopable kinds). */ checks: Predicate[] | undefined; /** The turn-scoped transcript (see `buildTurnTranscript`). */ transcript: IterationTranscript; } /** * Evaluate per-turn checks across a case's turns, reusing the same * {@link evaluatePredicates} engine against each turn's slice. Every result is * tagged with `scope: { kind: "turn", promptIndex }` so the UI and persisted * metadata can attribute it to the turn. Turns with no checks contribute * nothing. Order is preserved (turn order, then check order within a turn). * * Defense in depth: non-turn-scopable kinds (e.g. `tokenBudgetUnder`) are * dropped here even though the backend rejects them at the write boundary — * the evaluator must never silently treat a case-only check as turn-scoped if * one reaches it directly (a different write path, a test, a future caller). */ declare function evaluateTurnChecks(turns: TurnChecksInput[]): PredicateResult[]; /** * Argument matching for the `toolCalledWith` predicate. * * Delegates to the same `evaluateToolCalls` engine the tool-call matcher uses, * exposing all three `argumentMatching` modes (`exact` | `partial` | `ignore`). * Reusing one engine guarantees a predicate's notion of "these args match" is * identical to the existing `expectedToolCalls` matcher's. */ /** * True iff `actualArgs` satisfies `matcher` under its `argumentMatching` mode * (default `"partial"`). */ declare function argMatch(matcher: ArgMatcher, actualArgs: Record): boolean; /** * Adapter: build the stable {@link IterationTranscript} the predicate evaluator * consumes from the data an eval runner already has per iteration — the trace * (messages + spans), the tool calls, and token usage. * * Tool-error classification (content-error vs protocol-error) is delegated to * `extractToolErrors` so it stays in lockstep with the runner's existing * `traceIndicatesToolExecutionFailure` gate. */ /** Text of the last assistant message in a message list, if any. */ declare function extractFinalAssistantMessage(messages: unknown): string | undefined; interface BuildTranscriptInput { trace?: EvalTraceInput; toolCalls: TranscriptToolCall[]; usage?: TranscriptUsage; /** Override the message-derived final assistant text when the runner has it. */ finalAssistantMessage?: string; /** Widget render observation summaries, when the runner captured any. */ renderObservations?: RenderObservationSummary[]; /** * Tool errors the runner observed outside the trace. A model-free pinned * tool call has no trace for `extractToolErrors` to read, so its failures * (content-error / protocol-error) must be passed explicitly — otherwise * `noToolErrors` would pass falsely. Merged with trace-derived errors. */ toolErrors?: ToolErrorRecord[]; /** * User turns for the iteration, when the caller already counted them. * Otherwise derived from the trace's user-role messages. */ turnCount?: number; } /** * Per-turn signals the runner captured for a single prompt turn. The runner * already groups tool calls / assistant message / render observations by turn * (`promptSummaries`); this is the slice handed to per-turn check evaluation. */ interface TurnTranscriptInput { /** Tool calls observed during this turn only. */ toolCalls: TranscriptToolCall[]; /** This turn's assistant message text, if any. */ finalAssistantMessage?: string; /** Tool errors observed during this turn only. */ toolErrors?: ToolErrorRecord[]; /** Widget render observations recorded during this turn only. */ renderObservations?: RenderObservationSummary[]; /** Token usage for this turn, if measured (rarely available per-turn). */ usage?: TranscriptUsage; } /** * Assemble a turn-scoped {@link IterationTranscript} from per-turn signals. * Unlike {@link buildIterationTranscript} there is no trace to parse — the * runner supplies already-extracted per-turn data. Feeding this slice to the * existing `evaluatePredicates` makes "the final message", "the first tool", * "calls to X" all resolve to the turn, with no change to the evaluator core. */ declare function buildTurnTranscript(input: TurnTranscriptInput): IterationTranscript; /** Assemble an {@link IterationTranscript} from runner per-iteration data. */ declare function buildIterationTranscript(input: BuildTranscriptInput): IterationTranscript; export { ArgMatcher, type BuildTranscriptInput, IterationTranscript, Predicate, PredicateResult, RenderObservationSummary, ToolErrorRecord, TranscriptToolCall, TranscriptUsage, type TurnChecksInput, type TurnTranscriptInput, allPredicatesPassed, argMatch, buildIterationTranscript, buildTurnTranscript, evaluatePredicate, evaluatePredicates, evaluateTurnChecks, extractFinalAssistantMessage };