import { b as EvalTraceSpanInput, c as EvalExpectedToolCall, U as UserValueStage, d as StageState, F as FailureCategory, I as IterationStatus } from './index-fZyfLCHE.js'; import { P as PredicateScope, I as IterationTranscript, a as TranscriptUsage } from './types-BBk0lTBo.js'; import { z } from 'zod'; /** * Whether a scorer produced a number, blew up, was expected to run and didn't, * or simply does not apply. * * - `"scored"` — a value in [0,1] was produced. The only status that * carries a verdict. * - `"error"` — the scorer threw, timed out, or returned something * malformed. NEVER a low score: a crashed judge that reported `0` would be * indistinguishable from a judge that ran and disagreed. * - `"skipped"` — expected work that did not happen (the iteration * errored before scoring, a required input was missing). For a gating * scorer this fails closed by default: an unscored gate is not a passed * gate. * - `"not_applicable"` — the scorer does not apply to this iteration at all * (e.g. tool matching on a case that configured no expectations). NEVER * gates, and is excluded from every aggregation denominator — which is * exactly what distinguishes it from `"skipped"`. */ type ScoreStatus = "scored" | "error" | "skipped" | "not_applicable"; /** * Whether a scorer's verdict decides the iteration. * * - `"gating"` — a failing (or, per policy, an errored/skipped) score fails * the iteration. * - `"advisory"` — reported and rendered, never consulted for `passed`. The * hosted stance for judges ("Never mutates the run's `passed`") is the * default here too. */ type ScorerRole = "gating" | "advisory"; /** * What a gating scorer's non-verdict status does to the iteration. * `"fail"` is the fail-closed default for gating scorers; `"ignore"` is the * default for advisory ones, whose statuses never gate anyway. */ type ScorerErrorPolicy = "fail" | "ignore"; /** * Where an explicit `scorerId` came from. * * - `"explicit"` — the author named it. Stable across config edits, so it is * the only kind a CI gate or a cross-run diff may reference. * - `"generated"` — the runtime minted it positionally (e.g. * `predicate:responseContains#2`). **Positional and UNSTABLE**: inserting a * predicate above it renumbers it, so diffing and aggregation must not * track a generated id across config edits, and {@link GatePolicy}-style * selection by id must reject one. * - `"platform"` — the hosted platform minted it from a stable, non-positional * key (`predicate:`, `toolCalls:match`, `judge:goalCompletion`). * Stable across config edits like `"explicit"`, but nobody authored it, so a * report can still say where the id came from. */ type ScorerIdSource = "explicit" | "generated" | "platform"; /** * What a scorer is and how its verdict is treated — the authored half of the * contract. * * `implementationHash` is REQUIRED, and it is the field that makes this shape * honest. `scorerVersion` is author-maintained and therefore forgettable: two * judges whose prompts differ would hash identically unless someone remembered * to bump it. `implementationHash` is derived from what the scorer actually * does — the canonicalized predicate, the canonicalized prompt/rubric plus * template version, or an author-supplied config hash for a custom scorer — so * a changed prompt always changes the evaluation config hash. */ type ScoreDefinition = { /** Stable, author-facing identifier. Max 128 chars. */ scorerId: string; /** Whether {@link scorerId} is author-chosen or positionally generated. */ idSource: ScorerIdSource; /** Author-maintained version of the scorer's semantics. */ scorerVersion: string; /** * Digest of what the scorer actually does. See the type docs — this is not * optional and not a convenience. */ implementationHash: string; /** Human label for dashboards. Never load-bearing. */ label?: string; /** * True when the same transcript always yields the same value. Judges are * `false`; predicates, tool matching and the legacy boolean are `true`. * Read by the retry-exhausted path: deterministic scorers can still score a * partial transcript, non-deterministic ones are skipped. */ deterministic: boolean; /** * The bar a value must reach. Predicates / tool-match / legacy boolean use * `1`; judges default to `0.7` (the hosted `judgeConfig` default). */ passThreshold: number; role: ScorerRole; /** Defaults to `"fail"` when gating, `"ignore"` when advisory. */ onError?: ScorerErrorPolicy; /** * Defaults to `"fail"` when gating, `"ignore"` when advisory. Deliberately * separate from {@link onError}: "the judge crashed" and "the judge never * ran" are different failures and a team may reasonably tolerate one and not * the other. */ onSkipped?: ScorerErrorPolicy; /** Model string for non-deterministic scorers, e.g. `"anthropic/claude-sonnet-4-6"`. */ model?: string; /** Absent ⇒ case-level; `{ kind: "turn", promptIndex }` ⇒ per-turn. */ scope?: PredicateScope; }; /** * A {@link ScoreDefinition} with every semantic default filled in. * * This is the hashing form AND the wire form. Resolving before hashing is what * makes an omitted `onError` and an explicitly-configured `"fail"` hash * identically — without it, a no-op edit that spells out a default would read * as an evaluation-config change and flag every case as `configChanged`. */ type ResolvedScoreDefinition = Omit & { onError: ScorerErrorPolicy; onSkipped: ScorerErrorPolicy; }; /** * The join table gates and renderers read. * * Results carry only a `definitionHash`; role and error policies live here. A * consumer holding results alone cannot tell a gating failure from an advisory * one, which is why the public projection must expose BOTH. */ type EvaluationConfigSnapshot = { /** * `evaluationConfigHash` over the resolved definitions. Order-independent — * the hash sorts internally — so `definitions` may stay in authored order. */ hash: string; definitions: ResolvedScoreDefinition[]; }; /** * One scorer's verdict for one iteration. * * Deliberately does NOT repeat `role`, `onError` or `onSkipped`: duplicating * policy onto every row invites the two copies to disagree, and a disagreement * about whether something gates is precisely the disagreement you cannot * afford. Consumers join to the snapshot on {@link definitionHash}. */ type ScoreResult = { scorerId: string; scorerVersion: string; /** Joins this result to its definition in the run's snapshot. */ definitionHash: string; status: ScoreStatus; /** Present iff `status === "scored"`. Always within [0,1]. */ value?: number; /** Echoed from the definition so a row renders without the join. */ passThreshold: number; /** * DERIVED as `value >= passThreshold`, present iff `status === "scored"`. * Never model-asserted and never scorer-asserted — a judge that returns * `{score: 0.2, passed: true}` does not get to overrule the threshold. */ passed?: boolean; /** Free-text explanation. Truncated to 2000 chars by the producer. */ rationale?: string; /** Supporting snippets. At most 20 entries, each truncated to 300 chars. */ evidence?: string[]; deterministic: boolean; model?: string; /** Digest of the rendered judge prompt, for reproducibility triage. */ promptHash?: string; /** Present iff `status === "error"`. Truncated to 500 chars. */ error?: string; scope?: PredicateScope; }; /** * What a {@link Scorer} returns: an observation, never a finished verdict. * * Scorers cannot mint a {@link ScoreResult} directly — `passed` is derived and * bounds are enforced in exactly one place (`finalizeScoreResult`), so there is * no code path where a scorer asserts its own verdict. */ type ScoreRawOutcome = { kind: "scored"; /** Must be a finite number in [0,1]; anything else finalizes to `error`. */ value: number; rationale?: string; evidence?: string[]; model?: string; promptHash?: string; scope?: PredicateScope; } | { kind: "skipped"; rationale?: string; scope?: PredicateScope; } | { kind: "not_applicable"; rationale?: string; scope?: PredicateScope; }; /** * The versioned input a scorer grades against. * * Richer than {@link IterationTranscript}, which stays predicate-minimal by * design: predicates consume `context.transcript` and nothing else, while a * judge needs the message trace, the expectations and the usage totals. * * `version: 1` is a literal so a future `ScorerContextV2` can be discriminated * rather than sniffed. * * An `AbortSignal` is deliberately NOT a field here: signals are not data, they * do not serialize, and a context that cannot be written to a fixture is a * context nobody can test against. It is passed as a separate argument to * `score(context, signal)`. */ type ScorerContextV1 = { version: 1; scenario: { title: string; isNegativeTest?: boolean; /** Stable case identity when the caller knows it. */ scenarioKey?: string; }; /** Exactly what the deterministic predicates evaluate against. */ transcript: IterationTranscript; trace: { messages: Array<{ role: string; content: unknown; }>; spans?: EvalTraceSpanInput[]; }; expectedOutput?: string; expectedToolCalls?: EvalExpectedToolCall[]; usage?: TranscriptUsage; }; /** Max length of a `scorerId`. */ declare const MAX_SCORER_ID_LENGTH = 128; /** `rationale` is truncated to this many characters by the producer. */ declare const MAX_RATIONALE_LENGTH = 2000; /** `error` is truncated to this many characters by the producer. */ declare const MAX_ERROR_LENGTH = 500; /** At most this many `evidence` entries survive. */ declare const MAX_EVIDENCE_ENTRIES = 20; /** Each `evidence` entry is truncated to this many characters. */ declare const MAX_EVIDENCE_ENTRY_LENGTH = 300; /** * Version of the deterministic predicate evaluator, stamped as `scorerVersion` * on every predicate-derived definition. Bumping it is how a change in * predicate *evaluation semantics* (as opposed to a change in an individual * authored predicate, which `implementationHash` covers) reaches the * evaluation config hash. */ declare const PREDICATES_VERSION = "1"; /** * Deriving a stage's state from one run — the output side of the user-value * chain vocabulary pinned in `./chain.ts`. * * `./chain.ts` deliberately stops at the enums ("pinning the derivation output * belongs to whoever writes the derivation"). This module is that derivation: * the row shape, the reason codes, and the pure function that turns one * iteration's authored case + captured evidence into six stage rows. * * `deriveStageResults` is PURE and deterministic — no Convex ctx, no network, * no LLM, no clock. Same input, same six rows, forever. The validators live at * the bottom of the file and the analyzer functions themselves never touch `z`, * so they stay trivially unit-testable (the arrangement `analyzeSession` uses * in `mcpjam-backend`'s `convex/lib/sessionReadiness.ts`). * * Three rules this module exists to enforce, none of which are negotiable: * * 1. **Non-vacuity.** A stage reaches `passed` only when at least one piece * of eligible evidence was actually inspected. Zero evidence is * `notMeasured`. This is the whole point: a chain derived from missing * spans that quietly reads as green is worse than no chain at all. * 2. **`notReached` is derived from POSITION**, per `USER_VALUE_STAGES` * order — but only over a stage that measured NOTHING. A stage after the * first failure that has its own evidence keeps its own row: a run * disproves "never ran" the moment it produces a verdict for that stage. * The array is normative; this module never sorts it. * 3. **`evaluator` is never folded into another category.** A broken grader * is not a server defect, and counting it as one poisons every rate * derived from it. * * What this module deliberately does NOT do: * * - It never GUESSES `failureCategory: "metadata"` from the deterministic * evidence alone. That category means "tool names, descriptions or * schemas misled the model", which is a judgement about intent that no * span carries on its own. `categoryFor`'s `selection` branch below is * reachable, but ONLY through `evidence.metadataAttribution` — a scored, * evidence-carrying verdict from the D7 judge (attributed elsewhere: * `metadata-attribution` second-pass). No deterministic span or predicate * ever produces it. * - It never enforces policy. A policy block is REPRESENTED here * (`notMeasured` + `blockedByPolicy`); enforcing it belongs elsewhere. * - It never reads `finishReason`. That field is advisory display only and * must never feed a gate (see `EvalTraceSpan.finishReason`). */ /** * Bump when the derivation SEMANTICS change — not when a type moves. * * Stored on every derivation this module returns so a rebuild can target stale * rows (`stageAnalyzerVersion < CURRENT`). A versioned analyzer whose version * is not persisted cannot be recomputed selectively, which is the entire * reason `sessionReadiness` stamps `READINESS_ANALYZER_VERSION` on every * record it writes. */ declare const STAGE_ANALYZER_VERSION = 4; /** * Why a stage landed where it did. * * A closed vocabulary, for the same reason the states are: free-text reasons * cannot be aggregated, and "no evidence" versus "the executor emits no spans" * versus "an earlier stage failed" are three different operator actions. */ declare const STAGE_REASONS: readonly ["noSpanChannel", "noEvidenceCaptured", "matchVerdictUnavailable", "traceAbsent", "executorEmitsNoSpans", "blockedByPolicy", "evaluatorError", "setupAborted", "connectFailed", "toolsListFailed", "egressUnverified", "lifecycleStopped", "notAuthored", "earlierStageFailed", "missingToolCall", "unexpectedToolCall", "argumentMismatch", "toolError", "protocolError", "renderFailed", "predicateFailed", "observed", "impliedByLaterEvidence", "judgeObserved", "judgePartial", "judgeFailed", "judgePending", "judgeNotRequested"]; type StageReason = (typeof STAGE_REASONS)[number]; /** Pointers back at the evidence a row was decided from. */ type StageEvidenceRefs = { spanIds?: string[]; promptIndexes?: number[]; predicateReasons?: string[]; }; /** One stage's verdict for one iteration. */ type StageResultRow = { stage: UserValueStage; state: StageState; reason?: StageReason; evidence?: StageEvidenceRefs; }; /** The full derivation for one iteration. */ type StageDerivation = { /** ALWAYS six rows, in `USER_VALUE_STAGES` order. Never sorted. */ stageResults: StageResultRow[]; /** The FIRST failed stage, in chain order. Absent when nothing failed. */ firstFailedStage?: UserValueStage; /** * The bucket this iteration is grouped under. * * CONTRACT, and it matters to anyone aggregating these: this field is * "why there is no good outcome", NOT "which stage failed". It can be * present with `firstFailedStage` ABSENT — a setup abort is * `failureCategory: "setup"` with all six rows `notMeasured`, and an * evaluator error is `failureCategory: "evaluator"` the same way. Both are * real answers and both would be lost by omitting the category. A rate that * wants only measured server failures must therefore filter on * `firstFailedStage`, not on the presence of this field. Pinned by test. */ failureCategory?: FailureCategory; stageAnalyzerVersion: number; }; type StageSpanLike = { id?: string; category?: string; status?: string; toolName?: string; promptIndex?: number; mcpErrorCode?: number; }; type StagePromptSummaryLike = { promptIndex?: number; expectedToolCalls?: readonly unknown[]; missing?: readonly unknown[]; unexpected?: readonly unknown[]; argumentMismatches?: readonly unknown[]; /** * The turn's OWN verdict, under the match options the case authored. * * Load-bearing, and the reason this field exists: `unexpected` is populated * whenever an actual call went unmatched, but `maxExtraToolCalls` defaults to * `null` — extras are REPORTED and non-fatal (`evaluateToolCalls`). Deciding * `selection` from the raw field therefore reports `failed` for a run whose * verdict is `passed`, which is the common shape of an agentic multi-turn * case (a search call before the expected one). The turn already knows the * answer; this analyzer must not re-derive it. */ passed?: boolean; }; type StagePredicateResultLike = { passed?: boolean; reason?: string; }; type StageToolErrorLike = { kind?: string; toolName?: string; }; type StageRenderObservationLike = { status?: string; }; /** * The authored case — what makes `notApplicable` derivable. * * Without this the analyzer cannot tell "this stage does not apply to this * case" from "this stage was not measured", and every inapplicable stage would * be reported as an evidence gap. Authors never toggle stages: every field * here is INFERRED from what the case already declares. */ type StageAuthoredCase = { /** * `model_free` ⇒ no model ever chooses a tool ⇒ `selection` does not apply. * Inferred from the authored steps/turns (a case with no `prompt` step), * never authored directly. */ mode: "model_driven" | "model_free"; isNegativeTest?: boolean; /** The case authored at least one expected tool call. */ expectsToolCall?: boolean; /** The case asserts something about a rendered widget. */ expectsWidgetRender?: boolean; /** Count of authored user-value assertions (predicates, expectedOutput). */ assertionCount?: number; }; /** * One run-level setup phase (connect or tools/list), folded across every * configured target. Connect and tools-list happen once per run above the * iteration boundary, so this is the derivation input — not spans. */ type StageSetupPhaseSignal = { outcome: "ok" | "failed"; /** Present on a failure only. */ attribution?: "ours" | "theirs" | "unknown"; /** Positive canary evidence that our own egress works. */ egressVerified?: boolean; /** Culprit synthetic-span ids (`run-connect-` / `run-toolslist-`). */ spanIds?: string[]; }; type StageSetupSignals = { connection?: StageSetupPhaseSignal; discovery?: StageSetupPhaseSignal; }; /** Everything the run actually captured. */ type StageEvidence = { spans?: readonly StageSpanLike[]; /** * True when a trace object EXISTS but carries no span channel — the * caller-supplied `HostExecutor` case. Distinct from `spans: []`, and the * difference is the difference between "we looked and saw nothing happen" * and "this executor never reports what happened". */ traceLacksSpanChannel?: boolean; /** True when the iteration carries no trace at all. */ traceAbsent?: boolean; prompts?: readonly StagePromptSummaryLike[]; predicateResults?: readonly StagePredicateResultLike[]; toolErrors?: readonly StageToolErrorLike[]; renderObservations?: readonly StageRenderObservationLike[]; /** `tools_total_before` / `tools_exposed` — the one direct discovery signal. */ toolSignals?: { toolsTotalBefore?: number; toolsExposed?: number; }; /** * Structured connect / tools-list evidence, threaded per-iteration * (precedent: `toolSignals`). Synthetic `connection`/`discovery` spans * are persistence/timeline-only and never enter this field. */ setupSignals?: StageSetupSignals; /** The grader threw. Never folded into a server-side category. */ evaluatorErrored?: boolean; /** * Advisory judge evidence for this iteration. TIER 2: consulted only where * deterministic evidence is silent. Never overturns a predicate failure. */ judgeEvidence?: { status: "scored" | "error" | "skipped" | "not_applicable" | "pending"; /** * `pending` only: was a verdict ever actually owed? Drives the * `judgePending` vs `judgeNotRequested` split. */ pendingKind?: "scheduled" | "not_requested"; verdict?: "pass" | "partial" | "fail"; /** Bounded by the EXISTING evidence caps, same as predicate reasons. */ reasons?: readonly string[]; }; /** * D7's advisory judge: did the server's OWN tool metadata (names, * descriptions, schemas) mislead the model into a wrong or missing tool * choice? Same tier-2 shape as `judgeEvidence` — a report-only LLM * round trip consulted only where `selection` already failed * deterministically (`missingToolCall` / `unexpectedToolCall`). * * Answers a BINARY attribution question, not a graded band: there is no * `judgeEvidence`-style `verdict` scale here, because "did the metadata * cause this?" has no meaningful partial answer the way "did the user get * what they wanted?" does. */ metadataAttribution?: { status: "scored" | "error" | "skipped" | "not_applicable" | "pending"; /** `pending` only, same split as `judgeEvidence.pendingKind`. */ pendingKind?: "scheduled" | "not_requested"; /** `true` ⇒ the judge concluded the server's tool metadata caused the miss. */ attributed?: boolean; /** * Quoted evidence (description text vs. the ask) plus a one-line * rationale. Bounded by the EXISTING evidence caps, same as predicate * and judge reasons. */ reasons?: readonly string[]; }; }; type StageDerivationInput = { authored: StageAuthoredCase; evidence: StageEvidence; iteration: { status: IterationStatus; error?: string; }; /** D1 only REPRESENTS a policy block; enforcing one is a different step. */ policy?: { blocked: boolean; reason?: string; }; }; /** * Evidence bounds. * * A predicate `reason` is a judge rationale — graded CONTENT, of no fixed * length, already stored once under `metadata.predicates`. Copying it whole * into a second key doubles what the row retains and hands the redaction * contract a second place to reach. Bounded here, at the producer, so the * bound holds on every path rather than only where a validator happens to run. */ declare const MAX_EVIDENCE_REASONS = 5; declare const MAX_EVIDENCE_REASON_CHARS = 500; /** * Derive the six stage rows for one iteration. * * Pure and deterministic. Always returns exactly six rows in * `USER_VALUE_STAGES` order — position is how `notReached` is derived, so the * result must never be sorted or re-slotted by a caller. */ declare function deriveStageResults(input: StageDerivationInput): StageDerivation; declare const stageReasonSchema: z.ZodEnum<{ evaluatorError: "evaluatorError"; observed: "observed"; noSpanChannel: "noSpanChannel"; noEvidenceCaptured: "noEvidenceCaptured"; matchVerdictUnavailable: "matchVerdictUnavailable"; traceAbsent: "traceAbsent"; executorEmitsNoSpans: "executorEmitsNoSpans"; blockedByPolicy: "blockedByPolicy"; setupAborted: "setupAborted"; connectFailed: "connectFailed"; toolsListFailed: "toolsListFailed"; egressUnverified: "egressUnverified"; lifecycleStopped: "lifecycleStopped"; notAuthored: "notAuthored"; earlierStageFailed: "earlierStageFailed"; missingToolCall: "missingToolCall"; unexpectedToolCall: "unexpectedToolCall"; argumentMismatch: "argumentMismatch"; toolError: "toolError"; protocolError: "protocolError"; renderFailed: "renderFailed"; predicateFailed: "predicateFailed"; impliedByLaterEvidence: "impliedByLaterEvidence"; judgeObserved: "judgeObserved"; judgePartial: "judgePartial"; judgeFailed: "judgeFailed"; judgePending: "judgePending"; judgeNotRequested: "judgeNotRequested"; }>; declare const stageResultRowSchema: z.ZodObject<{ stage: z.ZodEnum<{ response: "response"; call: "call"; connection: "connection"; discovery: "discovery"; selection: "selection"; userValue: "userValue"; }>; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>; /** * A derivation as persisted. * * The `superRefine` is the load-bearing part: it re-asserts the two invariants * that make the rows readable at all — exactly six rows, in `USER_VALUE_STAGES` * order. A payload that arrives sorted alphabetically would otherwise validate * field-by-field while reporting a completely different set of blocked stages. */ declare const stageDerivationSchema: z.ZodObject<{ stageResults: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; stageAnalyzerVersion: z.ZodNumber; }, z.core.$strip>; /** * The metadata keys a derivation occupies on `testIteration.metadata`. * * Exported so every writer and every validator names them identically rather * than spelling the strings again. */ declare const STAGE_METADATA_KEYS: readonly ["stageResults", "firstFailedStage", "failureCategory", "stageAnalyzerVersion"]; /** Flatten a derivation into the metadata keys it persists under. */ declare function stageDerivationToMetadata(derivation: StageDerivation): Record; /** * The canonical **eval run decision summary** — one versioned shape that says * what a run decided, in what unit it counted, and what evidence sits under the * non-passing rows. * * This module is browser-safe and intentionally has no node-only deps. * * ── What this is for ───────────────────────────────────────────────────────── * * The six-stage user-value chain shipped as eval metadata, and every surface * then interpreted it independently: the API returned rows, the CLI counted * iterations and produced its own verdict, the Platform MCP server returned * neither, and a reader had to reconstruct the chain from raw tool calls. Three * readings of the same run is three chances to disagree about it. This contract * is the one reading. The API assembles it, Platform MCP returns it, the CLI * renders it, and JSON / JUnit / HTML all restate the same object. * * ── It EXPLAINS the verdict; it never DECIDES it ───────────────────────────── * * Nothing here aggregates trials into a verdict. Under verdict policy v2 the * authority is the run's own {@link EvalVerdictDecision}: its verdict, its * rates, its validity phase, its reasons, its per-case stability and mixed- * verdict flags are COPIED after validation and never recomputed from the * iteration rows. The one arithmetic this file performs on it is a tally of * `decision.cases[].verdict` into {@link EvalRunDecisionCounts}, and the schema * refuses a summary whose tally does not match the rows it claims to count — * so the counts cannot drift from the decision they summarize. * * A run that predates policy v2 may project `run.result` + `run.summary` * instead, and those counts are TRIALS. Policy-v2 counts are case-execution- * VARIANT aggregates. The two are different populations over the same run, and * {@link EvalRunDecisionCounts} therefore carries `measurementUnit` on every * count it ships. Calling both "cases" — which is what the surfaces did before * this contract — makes a 3-case suite with 5 repetitions report either 3 or 15 * depending on which surface you asked. * * ── `notEstablished` is a fourth verdict, not a spelling of `failed` ───────── * * `EVAL_RUN_VERDICTS` has three members because those are the three things a * DECIDED run can conclude. A run that is still going, or that stopped without * a decision, has concluded none of them, and this contract says so with a * fourth word plus an {@link EvalRunDecisionUndecidedReason}. Folding it into * `failed` reports a defect nothing observed; folding it into `inconclusive` * claims the validity phase ran and withheld a verdict, which it did not. * * ── Evidence is attached to the claim it supports ─────────────────────────── * * For a measured failure the evidence locator is read from the * `firstFailedStage` ROW ONLY. Unioning the span ids of the passing stages into * the failure explanation — which is what the previous per-case summary did — * hands an operator the spans of everything that worked and labels them as the * evidence for the thing that did not. A stage-less outcome (a setup abort, an * evaluator error) keeps a stage-less locator: there is a run, an iteration and * a trace to read, and inventing a stage to hang the link on would be a claim * about where the run broke that nothing established. * * ── No `.default()`, every object `.strict()` ──────────────────────────────── * * Same discipline as `./suite-file.ts` and `./verdict-policy.ts`, for the same * reasons: an omitted field stays omitted so the payload is byte-stable through * `canonicalJson`, and an unknown field is an error rather than a silent * passenger. */ /** * The contract version, as a literal. * * `1` because this shape has no predecessor on the wire: the SDK's older * `EvalDecisionSummary` was never versioned, never published through the API, * and never carried a schema field to bump. A consumer therefore reads * `schemaVersion` to know it is looking at this contract at all. */ declare const EVAL_RUN_DECISION_SUMMARY_SCHEMA_VERSION = 1; type EvalRunDecisionSummarySchemaVersion = typeof EVAL_RUN_DECISION_SUMMARY_SCHEMA_VERSION; /** The `$id` of the published JSON Schema for this contract. */ declare const EVAL_RUN_DECISION_SUMMARY_SCHEMA_ID = "https://mcpjam.com/schemas/eval-run-decision-summary/v1.json"; /** * What a run's decision summary is allowed to report. * * The first three are `EVAL_RUN_VERDICTS` verbatim — the same three words a * decided run concludes. The fourth is this contract's own: * * - `notEstablished` — no verdict exists to report. The run has not finished, * or it stopped without one, or its own decision could not be read. NOT a * failure and NOT `inconclusive`: `inconclusive` is a decision the validity * phase reached, and this is the absence of any decision at all. The * accompanying {@link EvalRunDecisionUndecided} says which. */ declare const EVAL_RUN_DECISION_VERDICTS: readonly ["passed", "failed", "inconclusive", "notEstablished"]; type EvalRunDecisionVerdict = (typeof EVAL_RUN_DECISION_VERDICTS)[number]; declare const evalRunDecisionVerdictSchema: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; notEstablished: "notEstablished"; }>; /** * Where the verdict came from — which is also which evidence a reader may * trust. * * - `policyV2` — the run's own {@link EvalVerdictDecision}. The summary * carries it verbatim, and `verdict` is its verdict. * - `legacy` — a percent-threshold run that predates policy v2. There is no * decision object to read, `verdict` is `run.result`, and any counts are * TRIALS. * - `none` — no verdict. `verdict` is `notEstablished` and `undecided` * says why. */ declare const EVAL_RUN_DECISION_VERDICT_SOURCES: readonly ["policyV2", "legacy", "none"]; type EvalRunDecisionVerdictSource = (typeof EVAL_RUN_DECISION_VERDICT_SOURCES)[number]; declare const evalRunDecisionVerdictSourceSchema: z.ZodEnum<{ legacy: "legacy"; none: "none"; policyV2: "policyV2"; }>; /** * What a count counts. Never inferred, never omitted from a count. * * - `caseVariant` — one case under one provider/model execution variant, as * aggregated by the run's own decision. This is the population policy v2 * decides over: repetitions are TRIALS inside one of these, not members of * it. * - `trial` — one iteration. What a legacy run's stored `summary` * counted, and what the per-iteration diagnostics below are. */ declare const EVAL_RUN_MEASUREMENT_UNITS: readonly ["caseVariant", "trial"]; type EvalRunMeasurementUnit = (typeof EVAL_RUN_MEASUREMENT_UNITS)[number]; declare const evalRunMeasurementUnitSchema: z.ZodEnum<{ caseVariant: "caseVariant"; trial: "trial"; }>; /** * Why no verdict was established. * * - `runNotTerminal` — the run is still pending or running. Poll it. * - `runStatusNotAVerdict` — a legacy run that stopped at `cancelled`, * `timed_out` or `failed`. Its stored summary describes the iterations it * happened to record, not the run it was asked to perform, so gating on it * is fail-open. (A policy-v2 run is NOT resolved this way — see * {@link assembleEvalRunDecisionSummary}.) * - `runResultNotAVerdict` — a legacy run that completed with no * recognizable `result`. * - `verdictSummaryUnavailable` — the run was decided under policy v2 and its * decision could not be read: absent, or refused by * {@link evalVerdictDecisionSchema}, or accompanied by the platform's own * integrity error (carried in `detail`). A partially-valid decision is * never published, so its absence is the whole answer. */ declare const EVAL_RUN_DECISION_UNDECIDED_REASONS: readonly ["runNotTerminal", "runStatusNotAVerdict", "runResultNotAVerdict", "verdictSummaryUnavailable"]; type EvalRunDecisionUndecidedReason = (typeof EVAL_RUN_DECISION_UNDECIDED_REASONS)[number]; declare const evalRunDecisionUndecidedReasonSchema: z.ZodEnum<{ runNotTerminal: "runNotTerminal"; runStatusNotAVerdict: "runStatusNotAVerdict"; runResultNotAVerdict: "runResultNotAVerdict"; verdictSummaryUnavailable: "verdictSummaryUnavailable"; }>; declare const evalRunDecisionUndecidedSchema: z.ZodObject<{ reason: z.ZodEnum<{ runNotTerminal: "runNotTerminal"; runStatusNotAVerdict: "runStatusNotAVerdict"; runResultNotAVerdict: "runResultNotAVerdict"; verdictSummaryUnavailable: "verdictSummaryUnavailable"; }>; detail: z.ZodOptional; }, z.core.$strict>; type EvalRunDecisionUndecided = z.infer; /** * A tally, with the population it tallied stated on it. * * A discriminated union rather than one object with optional members, so * `inconclusive` cannot appear on a trial count (a legacy run has no such * bucket) and cannot be omitted from a case-variant one (where it is a real * outcome, and dropping it silently moves unmeasured cases into neither * column). */ declare const evalRunDecisionCountsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{ measurementUnit: z.ZodLiteral<"caseVariant">; total: z.ZodNumber; passed: z.ZodNumber; failed: z.ZodNumber; inconclusive: z.ZodNumber; }, z.core.$strict>, z.ZodObject<{ measurementUnit: z.ZodLiteral<"trial">; total: z.ZodOptional; passed: z.ZodOptional; failed: z.ZodOptional; }, z.core.$strict>], "measurementUnit">; type EvalRunDecisionCounts = z.infer; /** * Whether this iteration's chain can be believed. * * - `verified` — the stored derivation validated against * {@link stageDerivationSchema}. Only this variant carries stages, a first * failed stage or a failure category. * - `unverified` — a derivation was stored and did not validate. The chain and * BOTH claims derived from it are withheld: `firstFailedStage` and * `failureCategory` are assertions ABOUT the rows, so rows that do not * validate leave nothing to check them against. Only the quarantine state * crosses, never the rejected claim. * - `absent` — no derivation was stored at all (an iteration predating the * analyzer). Distinct from `unverified`: nothing was rejected here, nothing * was ever offered. */ declare const evalRunDecisionChainSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{ status: z.ZodLiteral<"verified">; stages: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; analyzerVersion: z.ZodNumber; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"unverified">; analyzerVersion: z.ZodOptional; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"absent">; }, z.core.$strict>], "status">; type EvalRunDecisionChain = z.infer; /** * Where to go and look. * * `tracePath` is the trace endpoint's path RELATIVE TO THE API ROOT — the same * relative form `PlatformApiClient` takes, so it resolves against any * deployment's base URL rather than baking one host into a stored artifact. * * `stage` is present only when a first failed stage was established, and the * span ids / prompt indexes / reasons are then read from THAT ROW ALONE. */ declare const evalRunDecisionEvidenceSchema: z.ZodObject<{ runId: z.ZodString; iterationId: z.ZodString; stage: z.ZodOptional>; spanIds: z.ZodOptional>; promptIndexes: z.ZodOptional>; reasons: z.ZodOptional>; tracePath: z.ZodString; }, z.core.$strict>; type EvalRunDecisionEvidence = z.infer; /** * One non-passing iteration, as evidence beneath the run's verdict. * * These are TRIALS. They are never counted as cases and never override the * run's or a case's verdict — under policy v2 a case can pass with a failing * trial in it, and a reader who tallies these rows instead of reading * `decision.cases` has re-derived a different verdict from the same run. * * `caseId` is the case's SDK-DECLARED id when the run recorded one, and * `testCaseId` is the stored row id. They are kept apart because they are * different identities with different lifetimes. **Neither joins to * `decision.cases[].caseId`**, which is an ENCODED identity minted by the * platform from whichever spelling that run knew; matching on it here would * silently attach a trial to the wrong aggregate. */ declare const evalRunDecisionDiagnosticSchema: z.ZodObject<{ iterationId: z.ZodString; iterationNumber: z.ZodNumber; caseId: z.ZodOptional; testCaseId: z.ZodOptional; title: z.ZodOptional; status: z.ZodString; result: z.ZodOptional>; chain: z.ZodDiscriminatedUnion<[z.ZodObject<{ status: z.ZodLiteral<"verified">; stages: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; analyzerVersion: z.ZodNumber; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"unverified">; analyzerVersion: z.ZodOptional; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"absent">; }, z.core.$strict>], "status">; expected: z.ZodOptional; }, z.core.$strict>>; observed: z.ZodOptional>; failure: z.ZodOptional; }, z.core.$strict>>; evidence: z.ZodObject<{ runId: z.ZodString; iterationId: z.ZodString; stage: z.ZodOptional>; spanIds: z.ZodOptional>; promptIndexes: z.ZodOptional>; reasons: z.ZodOptional>; tracePath: z.ZodString; }, z.core.$strict>; nextAction: z.ZodString; }, z.core.$strict>; type EvalRunDecisionDiagnostic = z.infer; /** * One page of diagnostics, with its completeness stated rather than implied. * * `complete` is the load-bearing field and it means exactly one thing: `items` * is the WHOLE non-passing set for this run. A page reached through a cursor is * never complete, and neither is the last page of a walk that was cut short. A * partial page that claimed to be a complete failure list would let a reader * conclude "only these two cases failed" from a sample — the same confident- * verdict-about-page-one failure the CLI's iteration walk already guards. * * `scannedIterations` is how many iterations this page examined. It is what * separates "we looked at 50 and none of them failed" from "we did not look", * both of which otherwise render as an empty `items`. */ declare const evalRunDecisionDiagnosticsSchema: z.ZodObject<{ items: z.ZodArray; testCaseId: z.ZodOptional; title: z.ZodOptional; status: z.ZodString; result: z.ZodOptional>; chain: z.ZodDiscriminatedUnion<[z.ZodObject<{ status: z.ZodLiteral<"verified">; stages: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; analyzerVersion: z.ZodNumber; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"unverified">; analyzerVersion: z.ZodOptional; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"absent">; }, z.core.$strict>], "status">; expected: z.ZodOptional; }, z.core.$strict>>; observed: z.ZodOptional>; failure: z.ZodOptional; }, z.core.$strict>>; evidence: z.ZodObject<{ runId: z.ZodString; iterationId: z.ZodString; stage: z.ZodOptional>; spanIds: z.ZodOptional>; promptIndexes: z.ZodOptional>; reasons: z.ZodOptional>; tracePath: z.ZodString; }, z.core.$strict>; nextAction: z.ZodString; }, z.core.$strict>>; complete: z.ZodBoolean; nextCursor: z.ZodOptional; scannedIterations: z.ZodNumber; }, z.core.$strict>; type EvalRunDecisionDiagnostics = z.infer; declare const evalRunDecisionSummaryStructuralSchema: z.ZodObject<{ schemaVersion: z.ZodLiteral<1>; runId: z.ZodString; runStatus: z.ZodString; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; notEstablished: "notEstablished"; }>; verdictSource: z.ZodEnum<{ legacy: "legacy"; none: "none"; policyV2: "policyV2"; }>; counts: z.ZodOptional; total: z.ZodNumber; passed: z.ZodNumber; failed: z.ZodNumber; inconclusive: z.ZodNumber; }, z.core.$strict>, z.ZodObject<{ measurementUnit: z.ZodLiteral<"trial">; total: z.ZodOptional; passed: z.ZodOptional; failed: z.ZodOptional; }, z.core.$strict>], "measurementUnit">>; decision: z.ZodOptional; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; }>; reasons: z.ZodArray>; validity: z.ZodObject<{ policy: z.ZodObject<{ coverage: z.ZodDiscriminatedUnion<[z.ZodObject<{ kind: z.ZodLiteral<"allConfiguredTrialsAttempted">; minGradeableTrials: z.ZodLiteral<1>; }, z.core.$strict>, z.ZodObject<{ kind: z.ZodLiteral<"minEligibleTrials">; minEligibleTrials: z.ZodNumber; }, z.core.$strict>], "kind">; minCompletionRate: z.ZodNumber; maxEvaluatorErrorRate: z.ZodNumber; }, z.core.$strict>; holds: z.ZodBoolean; configuredTrials: z.ZodNumber; attemptedTrials: z.ZodNumber; eligibleTrials: z.ZodNumber; completionRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; evaluatorErrorRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; }, z.core.$strict>; cases: z.ZodArray; }, z.core.$strict>>; configuredTrials: z.ZodNumber; attemptedTrials: z.ZodNumber; eligibleTrials: z.ZodNumber; passedTrials: z.ZodNumber; failedTrials: z.ZodNumber; effectivePassThreshold: z.ZodNumber; passRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; completionRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; observedStability: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; mixedVerdict: z.ZodBoolean; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; }>; reason: z.ZodEnum<{ configuredTrialsNotAttempted: "configuredTrialsNotAttempted"; noGradeableTrials: "noGradeableTrials"; eligibleTrialsBelowMinimum: "eligibleTrialsBelowMinimum"; completionRateBelowMinimum: "completionRateBelowMinimum"; completionRateNotMeasured: "completionRateNotMeasured"; evaluatorErrorRateAboveMaximum: "evaluatorErrorRateAboveMaximum"; evaluatorErrorRateNotMeasured: "evaluatorErrorRateNotMeasured"; caseHasNoEligibleTrials: "caseHasNoEligibleTrials"; casePassRateMetThreshold: "casePassRateMetThreshold"; casePassRateBelowThreshold: "casePassRateBelowThreshold"; allMeasuredCasesMetThreshold: "allMeasuredCasesMetThreshold"; }>; }, z.core.$strict>>; }, z.core.$strict>>; undecided: z.ZodOptional; detail: z.ZodOptional; }, z.core.$strict>>; diagnostics: z.ZodObject<{ items: z.ZodArray; testCaseId: z.ZodOptional; title: z.ZodOptional; status: z.ZodString; result: z.ZodOptional>; chain: z.ZodDiscriminatedUnion<[z.ZodObject<{ status: z.ZodLiteral<"verified">; stages: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; analyzerVersion: z.ZodNumber; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"unverified">; analyzerVersion: z.ZodOptional; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"absent">; }, z.core.$strict>], "status">; expected: z.ZodOptional; }, z.core.$strict>>; observed: z.ZodOptional>; failure: z.ZodOptional; }, z.core.$strict>>; evidence: z.ZodObject<{ runId: z.ZodString; iterationId: z.ZodString; stage: z.ZodOptional>; spanIds: z.ZodOptional>; promptIndexes: z.ZodOptional>; reasons: z.ZodOptional>; tracePath: z.ZodString; }, z.core.$strict>; nextAction: z.ZodString; }, z.core.$strict>>; complete: z.ZodBoolean; nextCursor: z.ZodOptional; scannedIterations: z.ZodNumber; }, z.core.$strict>; }, z.core.$strict>; /** * The summary validator, with the cross-field rules that make the envelope * self-consistent. * * These are CONSISTENCY checks on a supplied summary, in the same spirit as * `evalVerdictDecisionSchema`: they refuse a payload whose headline does not * follow from the evidence shipped beside it. The one that matters most is the * count tally — it is what stops a renderer's "2/3 passed" from drifting away * from the decision it claims to be reading. */ declare const evalRunDecisionSummarySchema: z.ZodObject<{ schemaVersion: z.ZodLiteral<1>; runId: z.ZodString; runStatus: z.ZodString; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; notEstablished: "notEstablished"; }>; verdictSource: z.ZodEnum<{ legacy: "legacy"; none: "none"; policyV2: "policyV2"; }>; counts: z.ZodOptional; total: z.ZodNumber; passed: z.ZodNumber; failed: z.ZodNumber; inconclusive: z.ZodNumber; }, z.core.$strict>, z.ZodObject<{ measurementUnit: z.ZodLiteral<"trial">; total: z.ZodOptional; passed: z.ZodOptional; failed: z.ZodOptional; }, z.core.$strict>], "measurementUnit">>; decision: z.ZodOptional; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; }>; reasons: z.ZodArray>; validity: z.ZodObject<{ policy: z.ZodObject<{ coverage: z.ZodDiscriminatedUnion<[z.ZodObject<{ kind: z.ZodLiteral<"allConfiguredTrialsAttempted">; minGradeableTrials: z.ZodLiteral<1>; }, z.core.$strict>, z.ZodObject<{ kind: z.ZodLiteral<"minEligibleTrials">; minEligibleTrials: z.ZodNumber; }, z.core.$strict>], "kind">; minCompletionRate: z.ZodNumber; maxEvaluatorErrorRate: z.ZodNumber; }, z.core.$strict>; holds: z.ZodBoolean; configuredTrials: z.ZodNumber; attemptedTrials: z.ZodNumber; eligibleTrials: z.ZodNumber; completionRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; evaluatorErrorRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; }, z.core.$strict>; cases: z.ZodArray; }, z.core.$strict>>; configuredTrials: z.ZodNumber; attemptedTrials: z.ZodNumber; eligibleTrials: z.ZodNumber; passedTrials: z.ZodNumber; failedTrials: z.ZodNumber; effectivePassThreshold: z.ZodNumber; passRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; completionRate: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; observedStability: z.ZodDiscriminatedUnion<[z.ZodObject<{ state: z.ZodLiteral<"measured">; value: z.ZodNumber; numerator: z.ZodNumber; denominator: z.ZodNumber; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>, z.ZodObject<{ state: z.ZodLiteral<"notMeasured">; value: z.ZodNull; numerator: z.ZodLiteral<0>; denominator: z.ZodLiteral<0>; exclusions: z.ZodObject<{ notTerminal: z.ZodOptional; skipped: z.ZodOptional; setupFailed: z.ZodOptional; cancelled: z.ZodOptional; timedOut: z.ZodOptional; executionFailed: z.ZodOptional; evaluatorError: z.ZodOptional; }, z.core.$strict>; }, z.core.$strict>], "state">; mixedVerdict: z.ZodBoolean; verdict: z.ZodEnum<{ failed: "failed"; passed: "passed"; inconclusive: "inconclusive"; }>; reason: z.ZodEnum<{ configuredTrialsNotAttempted: "configuredTrialsNotAttempted"; noGradeableTrials: "noGradeableTrials"; eligibleTrialsBelowMinimum: "eligibleTrialsBelowMinimum"; completionRateBelowMinimum: "completionRateBelowMinimum"; completionRateNotMeasured: "completionRateNotMeasured"; evaluatorErrorRateAboveMaximum: "evaluatorErrorRateAboveMaximum"; evaluatorErrorRateNotMeasured: "evaluatorErrorRateNotMeasured"; caseHasNoEligibleTrials: "caseHasNoEligibleTrials"; casePassRateMetThreshold: "casePassRateMetThreshold"; casePassRateBelowThreshold: "casePassRateBelowThreshold"; allMeasuredCasesMetThreshold: "allMeasuredCasesMetThreshold"; }>; }, z.core.$strict>>; }, z.core.$strict>>; undecided: z.ZodOptional; detail: z.ZodOptional; }, z.core.$strict>>; diagnostics: z.ZodObject<{ items: z.ZodArray; testCaseId: z.ZodOptional; title: z.ZodOptional; status: z.ZodString; result: z.ZodOptional>; chain: z.ZodDiscriminatedUnion<[z.ZodObject<{ status: z.ZodLiteral<"verified">; stages: z.ZodArray; state: z.ZodEnum<{ failed: "failed"; passed: "passed"; notReached: "notReached"; notMeasured: "notMeasured"; notApplicable: "notApplicable"; }>; reason: z.ZodOptional>; evidence: z.ZodOptional>; promptIndexes: z.ZodOptional>; predicateReasons: z.ZodOptional>; }, z.core.$strip>>; }, z.core.$strip>>; firstFailedStage: z.ZodOptional>; failureCategory: z.ZodOptional>; analyzerVersion: z.ZodNumber; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"unverified">; analyzerVersion: z.ZodOptional; analyzerVersionAhead: z.ZodOptional>; }, z.core.$strict>, z.ZodObject<{ status: z.ZodLiteral<"absent">; }, z.core.$strict>], "status">; expected: z.ZodOptional; }, z.core.$strict>>; observed: z.ZodOptional>; failure: z.ZodOptional; }, z.core.$strict>>; evidence: z.ZodObject<{ runId: z.ZodString; iterationId: z.ZodString; stage: z.ZodOptional>; spanIds: z.ZodOptional>; promptIndexes: z.ZodOptional>; reasons: z.ZodOptional>; tracePath: z.ZodString; }, z.core.$strict>; nextAction: z.ZodString; }, z.core.$strict>>; complete: z.ZodBoolean; nextCursor: z.ZodOptional; scannedIterations: z.ZodNumber; }, z.core.$strict>; }, z.core.$strict>; type EvalRunDecisionSummary = z.infer; /** The already-public run fields a summary reads. Nothing else. */ type EvalRunDecisionRunInput = { id: string; status: string; result?: string | null; summary?: { total?: number; passed?: number; failed?: number; } | null; verdictPolicyVersion?: unknown; verdictSummary?: unknown; verdictPolicyIntegrityError?: unknown; }; /** The already-public iteration fields a diagnostic reads. Nothing else. */ type EvalRunDecisionIterationInput = { id: string; iterationNumber: number; status: string; result?: string | null; /** The case's SDK-declared id, when the run recorded one. */ caseId?: string | null; /** The stored case row id. */ testCaseId?: string | null; title?: string | null; expectedToolCalls?: readonly unknown[]; actualToolCalls?: readonly unknown[]; error?: string | null; stageResults?: unknown; firstFailedStage?: unknown; failureCategory?: unknown; stageAnalyzerVersion?: unknown; stageResultsUnverified?: unknown; }; type EvalRunDecisionAssemblyInput = { /** Needed only to build the trace path; never stored on the summary. */ projectId: string; run: EvalRunDecisionRunInput; /** ONE page of iterations, in the order the API returned them. */ iterations: readonly EvalRunDecisionIterationInput[]; page: { /** True only when `iterations` is the run's whole set. */ complete: boolean; nextCursor?: string; }; }; /** * Build the canonical summary from one already-projected run and one page of * already-projected iterations. * * PURE: no network, no pagination, no clock. The API route and the SDK's * compatibility fallback both call THIS function, which is what makes their * output byte-equivalent for the same input — the alternative, two assemblers * that agree today, is the drift this whole lane exists to remove. */ declare function assembleEvalRunDecisionSummary(input: EvalRunDecisionAssemblyInput): EvalRunDecisionSummary; /** * The iteration-trace endpoint, relative to the API root. * * One definition, so the path a summary hands a reader is the same path the * SDK client would call. Exported because the CLI prints it and the docs cite * it. */ declare function evalIterationTracePath(projectId: string, runId: string, iterationId: string): string; /** The failure category a diagnostic was grouped under, when one was verified. */ declare function decisionDiagnosticFailureCategory(diagnostic: EvalRunDecisionDiagnostic): FailureCategory | undefined; /** The first failed stage a diagnostic established, when one was verified. */ declare function decisionDiagnosticFirstFailedStage(diagnostic: EvalRunDecisionDiagnostic): UserValueStage | undefined; /** @see EVAL_RUN_DECISION_VERDICTS */ declare const EVAL_RUN_DECISION_VERDICT_LABELS: Readonly<{ passed: string; failed: string; inconclusive: string; notEstablished: string; }>; /** @see EVAL_RUN_DECISION_VERDICT_SOURCES */ declare const EVAL_RUN_DECISION_VERDICT_SOURCE_LABELS: Readonly<{ policyV2: string; legacy: string; none: string; }>; /** * @see EVAL_RUN_MEASUREMENT_UNITS * * Singular and plural, because a count is always rendered next to its unit and * "1 case variants" is the kind of wrongness that makes a reader distrust the * number beside it. */ declare const EVAL_RUN_MEASUREMENT_UNIT_LABELS: Readonly<{ caseVariant: { one: string; many: string; }; trial: { one: string; many: string; }; }>; /** @see EVAL_RUN_DECISION_UNDECIDED_REASONS */ declare const EVAL_RUN_DECISION_UNDECIDED_REASON_LABELS: Readonly<{ runNotTerminal: string; runStatusNotAVerdict: string; runResultNotAVerdict: string; verdictSummaryUnavailable: string; }>; /** The unit's word for `count`, singular or plural. */ declare function measurementUnitLabel(unit: EvalRunMeasurementUnit, count: number): string; export { type StagePromptSummaryLike as $, type EvalRunDecisionVerdict as A, type EvalRunDecisionVerdictSource as B, type EvalRunMeasurementUnit as C, MAX_EVIDENCE_ENTRIES as D, type EvaluationConfigSnapshot as E, MAX_EVIDENCE_ENTRY_LENGTH as F, MAX_EVIDENCE_REASONS as G, MAX_EVIDENCE_REASON_CHARS as H, MAX_RATIONALE_LENGTH as I, MAX_SCORER_ID_LENGTH as J, STAGE_ANALYZER_VERSION as K, STAGE_METADATA_KEYS as L, MAX_ERROR_LENGTH as M, STAGE_REASONS as N, type ScoreStatus as O, PREDICATES_VERSION as P, type ScorerContextV1 as Q, type ResolvedScoreDefinition as R, type ScoreResult as S, type ScorerErrorPolicy as T, type ScorerIdSource as U, type StageAuthoredCase as V, type StageDerivation as W, type StageDerivationInput as X, type StageEvidence as Y, type StageEvidenceRefs as Z, type StagePredicateResultLike as _, type ScoreDefinition as a, type StageReason as a0, type StageRenderObservationLike as a1, type StageSetupPhaseSignal as a2, type StageSetupSignals as a3, type StageSpanLike as a4, type StageToolErrorLike as a5, assembleEvalRunDecisionSummary as a6, decisionDiagnosticFailureCategory as a7, decisionDiagnosticFirstFailedStage as a8, deriveStageResults as a9, evalIterationTracePath as aa, evalRunDecisionChainSchema as ab, evalRunDecisionCountsSchema as ac, evalRunDecisionDiagnosticSchema as ad, evalRunDecisionDiagnosticsSchema as ae, evalRunDecisionEvidenceSchema as af, evalRunDecisionSummarySchema as ag, evalRunDecisionSummaryStructuralSchema as ah, evalRunDecisionUndecidedReasonSchema as ai, evalRunDecisionUndecidedSchema as aj, evalRunDecisionVerdictSchema as ak, evalRunDecisionVerdictSourceSchema as al, evalRunMeasurementUnitSchema as am, measurementUnitLabel as an, stageDerivationSchema as ao, stageDerivationToMetadata as ap, stageReasonSchema as aq, stageResultRowSchema as ar, type ScoreRawOutcome as b, type StageResultRow as c, type EvalRunDecisionSummary as d, type ScorerRole as e, EVAL_RUN_DECISION_SUMMARY_SCHEMA_ID as f, EVAL_RUN_DECISION_SUMMARY_SCHEMA_VERSION as g, EVAL_RUN_DECISION_UNDECIDED_REASONS as h, EVAL_RUN_DECISION_UNDECIDED_REASON_LABELS as i, EVAL_RUN_DECISION_VERDICTS as j, EVAL_RUN_DECISION_VERDICT_LABELS as k, EVAL_RUN_DECISION_VERDICT_SOURCES as l, EVAL_RUN_DECISION_VERDICT_SOURCE_LABELS as m, EVAL_RUN_MEASUREMENT_UNITS as n, EVAL_RUN_MEASUREMENT_UNIT_LABELS as o, type EvalRunDecisionAssemblyInput as p, type EvalRunDecisionChain as q, type EvalRunDecisionCounts as r, type EvalRunDecisionDiagnostic as s, type EvalRunDecisionDiagnostics as t, type EvalRunDecisionEvidence as u, type EvalRunDecisionIterationInput as v, type EvalRunDecisionRunInput as w, type EvalRunDecisionSummarySchemaVersion as x, type EvalRunDecisionUndecided as y, type EvalRunDecisionUndecidedReason as z };