import { z } from 'zod'; import { a as EvalMatchOptions } from './index-fZyfLCHE.js'; /** * State-based predicate system for deterministic eval gating. * * A {@link Predicate} is a declarative assertion over a single iteration's * transcript. Predicates are the **gate**: a pure function of the transcript * yields the same verdict every time (same transcript → same result), which is * the property a CI release gate requires and a stochastic LLM judge cannot * provide. The `serverQuality` LLM judge remains the advisory **insight** layer. * * The union is intentionally small (12 types). It grows only when a real corpus * task demands a new one — not speculatively. * * Hosted in `@mcpjam/sdk` (browser-safe; reuses the `../matchers` argument * engine) so the inspector GUI runner and the `mcpjam cloud eval` CLI share one * implementation. */ /** * Argument-matching mode reused from the eval matcher * (`EvalMatchOptions.argumentMatching`): * * - `"partial"` (default) — only the keys present in `args` are checked; * the actual call may carry extra keys; placeholder strings like * `"string"`/`"number"`/`"any"` are interpreted as type checks. * - `"exact"` — deep equality on the args object; no extras, no placeholders. * - `"ignore"` — arguments are not compared (only the tool name matters). */ type ArgMatchMode = NonNullable; /** * Expected-argument matcher for {@link Predicate} `toolCalledWith`. * * `args` is the expected argument shape; `argumentMatching` selects the * semantics. Reuses the exact same engine as the tool-call matcher so a * predicate and the existing `expectedToolCalls` matcher agree on what * "these args match" means. */ type ArgMatcher = { args: Record; /** Defaults to `"partial"` when omitted. */ argumentMatching?: ArgMatchMode; }; /** * The deterministic predicate library. * * Discriminated on `type`. Each variant is evaluated by a pure function over * the {@link IterationTranscript}. */ type Predicate = /** A call to `toolName` whose args satisfy `args` occurred at least `minCount` (default 1) times. */ { type: "toolCalledWith"; toolName: string; args: ArgMatcher; minCount?: number; } /** `toolName` was called at least once (args irrelevant). */ | { type: "toolCalledAtLeastOnce"; toolName: string; } /** `toolName` was never called (forbidden tool). */ | { type: "toolNeverCalled"; toolName: string; } /** The first tool call observed in the transcript was `toolName`. */ | { type: "firstToolWas"; toolName: string; } /** The final assistant message contains `needle`. Case-insensitive unless `caseSensitive`. */ | { type: "responseContains"; needle: string; caseSensitive?: boolean; } /** The final assistant message matches the regular expression `pattern` (regex source, no flags). */ | { type: "responseMatches"; pattern: string; } /** No tool produced an error (neither MCP `isError: true` nor a JSON-RPC/transport failure). */ | { type: "noToolErrors"; } /** The final assistant message is a non-empty (non-whitespace) string. */ | { type: "finalAssistantMessageNonEmpty"; } /** Total token usage for the iteration is strictly under `tokens`. */ | { type: "tokenBudgetUnder"; tokens: number; } /** * At least one widget render observation (narrowed to `toolName` when set) * has `status === "rendered"`. Fails closed when the iteration recorded no * render observations in scope. */ | { type: "widgetRendered"; toolName?: string; } /** * Every rendered widget observation (narrowed to `toolName` when set) mounted * in strictly under `ms` milliseconds. Fails closed when no observation in * scope rendered — an unrendered widget has no latency to attest. */ | { type: "widgetRenderLatencyUnder"; ms: number; toolName?: string; } /** * No widget render observation (narrowed to `toolName` when set) captured * console errors. Fails closed when the iteration recorded no render * observations in scope. */ | { type: "widgetNoConsoleErrors"; toolName?: string; } /** * The iteration used STRICTLY FEWER than `turns` user turns. * * A "turn" is one user-role message in the transcript, so this expresses * "resolved in under N turns" without conflating it with a run's `maxTurns` * cap — that bounds the agent loop, this grades the outcome. Fails closed * when the transcript carries no turn count: an unmeasured budget is not a * met budget. */ | { type: "turnCountUnder"; turns: number; }; /** The `type` discriminants of {@link Predicate}, for validators. */ type PredicateType = Predicate["type"]; /** * Predicate kinds that may be authored on an individual prompt turn (evaluated * against that turn's slice of the transcript). Every kind is turn-scopable * EXCEPT `tokenBudgetUnder` and `turnCountUnder`: per-turn token usage is not * reliably captured, and a turn count evaluated against a single turn's slice * is always 1 — both are whole-iteration (case-level) concerns. * * Mirrored in `mcpjam-backend/convex/lib/predicates.ts` * (`TURN_SCOPABLE_PREDICATE_KINDS`). Used by the per-turn "Add check" menu and * the backend write-time guard. */ declare const TURN_SCOPABLE_PREDICATE_KINDS: readonly ["toolCalledWith", "toolCalledAtLeastOnce", "toolNeverCalled", "firstToolWas", "responseContains", "responseMatches", "noToolErrors", "finalAssistantMessageNonEmpty", "widgetRendered", "widgetRenderLatencyUnder", "widgetNoConsoleErrors"]; declare function isTurnScopablePredicateKind(kind: string): boolean; /** * Placeholder strings the matcher's `partial` mode treats as type checks * instead of literal equality. Exposed as a Zod literal union so authoring * UIs can offer them as drop-down options when constructing arg matchers. * * The actual leaf may also be any JSON literal (string/number/boolean/object/ * array/null) — that's not captured here because the args blob is * `z.record(z.string(), z.unknown())` at the wire boundary. */ declare const PREDICATE_PLACEHOLDER_STRINGS: readonly ["any", "string", "number", "boolean", "object", "array", "null"]; /** Zod schema for {@link ArgMatcher}. `args` is unrestricted JSON. */ declare const argMatcherSchema: z.ZodObject<{ args: z.ZodRecord; argumentMatching: z.ZodOptional>; }, z.core.$strip>; /** * Zod schema for {@link Predicate}. Uses `z.discriminatedUnion` on `type` * so authoring-side validation surfaces a precise error (e.g. "unknown * predicate type 'firstToolWass'") rather than a generic union failure. */ declare const predicateSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{ type: z.ZodLiteral<"toolCalledWith">; toolName: z.ZodString; args: z.ZodObject<{ args: z.ZodRecord; argumentMatching: z.ZodOptional>; }, z.core.$strip>; minCount: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolCalledAtLeastOnce">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolNeverCalled">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"firstToolWas">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseContains">; needle: z.ZodString; caseSensitive: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseMatches">; pattern: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"noToolErrors">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"finalAssistantMessageNonEmpty">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"tokenBudgetUnder">; tokens: z.ZodNumber; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRendered">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRenderLatencyUnder">; ms: z.ZodNumber; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetNoConsoleErrors">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"turnCountUnder">; turns: z.ZodNumber; }, z.core.$strip>], "type">; /** Array of predicates — used for both suite defaults and case overrides. */ declare const predicateArraySchema: z.ZodArray; toolName: z.ZodString; args: z.ZodObject<{ args: z.ZodRecord; argumentMatching: z.ZodOptional>; }, z.core.$strip>; minCount: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolCalledAtLeastOnce">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolNeverCalled">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"firstToolWas">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseContains">; needle: z.ZodString; caseSensitive: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseMatches">; pattern: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"noToolErrors">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"finalAssistantMessageNonEmpty">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"tokenBudgetUnder">; tokens: z.ZodNumber; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRendered">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRenderLatencyUnder">; ms: z.ZodNumber; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetNoConsoleErrors">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"turnCountUnder">; turns: z.ZodNumber; }, z.core.$strip>], "type">>; /** * Case-level predicate override envelope. The {@link mode} eliminates the * `predicates`/`additionalPredicates` ambiguity (see plan Phase 2): * * - `inherit` — effective predicates = suite defaults (`list` ignored). * - `replace` — effective predicates = `list`. * - `extend` — effective predicates = suite defaults followed by `list`. */ declare const casePredicatesSchema: z.ZodObject<{ mode: z.ZodEnum<{ inherit: "inherit"; replace: "replace"; extend: "extend"; }>; list: z.ZodArray; toolName: z.ZodString; args: z.ZodObject<{ args: z.ZodRecord; argumentMatching: z.ZodOptional>; }, z.core.$strip>; minCount: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolCalledAtLeastOnce">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"toolNeverCalled">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"firstToolWas">; toolName: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseContains">; needle: z.ZodString; caseSensitive: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"responseMatches">; pattern: z.ZodString; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"noToolErrors">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"finalAssistantMessageNonEmpty">; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"tokenBudgetUnder">; tokens: z.ZodNumber; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRendered">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetRenderLatencyUnder">; ms: z.ZodNumber; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"widgetNoConsoleErrors">; toolName: z.ZodOptional; }, z.core.$strip>, z.ZodObject<{ type: z.ZodLiteral<"turnCountUnder">; turns: z.ZodNumber; }, z.core.$strip>], "type">>; }, z.core.$strip>; type CasePredicates = z.infer; type PredicatePlaceholder = (typeof PREDICATE_PLACEHOLDER_STRINGS)[number]; /** * How a tool failure surfaced. The plan requires `noToolErrors` to distinguish * these two cases (and report which fired), matching the runner's existing * `traceIndicatesToolExecutionFailure` gate, which treats both as failures: * * - `"content-error"` — an MCP `CallToolResult` with `isError: true`. The * tool executed and reported a domain error the protocol-correct way. * - `"protocol-error"` — a JSON-RPC / transport-level failure (the AI SDK * `tool-error` stream part, or an errored tool span). The call itself * failed; no protocol-correct result was produced. */ type ToolErrorKind = "content-error" | "protocol-error"; /** A single detected tool failure, used by the `noToolErrors` predicate. */ type ToolErrorRecord = { toolName?: string; kind: ToolErrorKind; /** Optional human-readable detail surfaced in the predicate reason. */ message?: string; }; /** Token usage totals for an iteration. */ type TranscriptUsage = { inputTokens?: number; outputTokens?: number; totalTokens?: number; }; /** A tool call observed in the transcript: `{ toolName, arguments }`. */ type TranscriptToolCall = { toolName: string; arguments: Record; }; /** * Outcome states of an MCP App widget render attempt. Hand-mirror of the * inspector's `EvalTraceWidgetRenderStatus` (shared/eval-trace.ts) — the SDK * stays import-free of the inspector app, same arrangement as the Convex * validator mirror. Only `"rendered"` means success; every other literal names * the stage that failed. */ type RenderObservationStatus = "rendered" | "no_ui_resource" | "resource_read_failed" | "mount_failed" | "bridge_timeout" | "render_error" | "blank_screenshot" | "screenshot_failed" | "browser_unavailable"; /** * Screenshot-free summary of one widget render observation, carried on the * transcript for the `widget*` predicates. The runner maps its richer * `RunnerWidgetRenderObservation` (base64 screenshot, blocked requests, …) * down to this shape; fixtures author it directly. */ type RenderObservationSummary = { toolCallId?: string; toolName: string; serverId?: string; status: RenderObservationStatus; elapsedMs: number; consoleErrors?: string[]; }; /** * The stable input shape predicates evaluate against. * * Deliberately minimal: it carries exactly what the 8 V1 predicates need and * nothing else, so it can be produced both by the live eval runner (which maps * its internal per-iteration state onto this shape) and by hand-authored test * fixtures. New predicates that need more signal extend this type. */ type IterationTranscript = { /** Ordered tool calls across all turns of the iteration. */ toolCalls: TranscriptToolCall[]; /** Tool failures detected over the iteration trace. Absent/empty ⇒ no errors. */ toolErrors?: ToolErrorRecord[]; /** Text of the final assistant message of the iteration, if any. */ finalAssistantMessage?: string; /** Token usage totals for the whole iteration, if measured. */ usage?: TranscriptUsage; /** * Widget render observations recorded over the iteration, if any. Absent ⇒ * the `widget*` predicates fail closed (no signal is not a pass). */ renderObservations?: RenderObservationSummary[]; /** * User turns observed over the iteration — user-role messages in the * transcript. Absent ⇒ `turnCountUnder` fails closed, same rule as * `usage` and `tokenBudgetUnder`. */ turnCount?: number; }; /** * Where a check runs ("scope"). Absent ⇒ the check is case-level (evaluated * against the whole-iteration transcript). `{ kind: "turn", promptIndex }` * marks a check authored on a single prompt turn and evaluated against that * turn's slice of the transcript. An object (not a bare index) so future scope * kinds can be added without a wire break. * * Mirrored in `mcpjam-backend/convex/lib/predicates.ts` (`PredicateScope`). */ type PredicateScope = { kind: "turn"; promptIndex: number; }; /** Zod schema for {@link PredicateScope}. */ declare const predicateScopeSchema: z.ZodObject<{ kind: z.ZodLiteral<"turn">; promptIndex: z.ZodNumber; }, z.core.$strip>; /** Per-predicate verdict row, persisted to `testIteration.metadata.predicates`. */ type PredicateResult = { predicate: Predicate; passed: boolean; /** Structured, deterministic explanation — names the expected vs actual on failure. */ reason: string; /** Absent ⇒ case-level; `{ kind: "turn", promptIndex }` ⇒ per-turn. */ scope?: PredicateScope; }; export { type ArgMatcher as A, type CasePredicates as C, type IterationTranscript as I, type PredicateScope as P, type RenderObservationSummary as R, type ToolErrorRecord as T, type TranscriptUsage as a, type Predicate as b, type PredicateResult as c, type TranscriptToolCall as d, type ArgMatchMode as e, PREDICATE_PLACEHOLDER_STRINGS as f, type PredicatePlaceholder as g, type PredicateType as h, type RenderObservationStatus as i, TURN_SCOPABLE_PREDICATE_KINDS as j, type ToolErrorKind as k, argMatcherSchema as l, casePredicatesSchema as m, isTurnScopablePredicateKind as n, predicateSchema as o, predicateArraySchema as p, predicateScopeSchema as q };