import { z } from 'zod'; import { S as ScoreResult, E as EvaluationConfigSnapshot, a as ScoreDefinition, R as ResolvedScoreDefinition, b as ScoreRawOutcome } from './decision-summary-CksWu77D.js'; /** * Canonical JSON + SHA-256, the hashing primitive under the evaluation * contract. * * This module is browser-safe and intentionally has no node-only deps. * * Every hash in the contract must be reproducible in FOUR runtimes that share * no code: the SDK (Node), the inspector client (browser), the inspector server * and the Convex default runtime (where the backend hand-mirrors this file, * because `convex/` may not import the SDK main entry). That constraint drives * two choices: * * - **Canonicalization is RFC 8785-style, not `JSON.stringify`.** Key order * is an accident of construction; `{a,b}` and `{b,a}` describe the same * evaluation config and must digest identically, or a cosmetic refactor * would mark every case `configChanged`. * - **The digest is `@noble/hashes` SHA-256, not Web Crypto.** Web Crypto's * `subtle.digest` is async, and an async digest would poison every pure * derivation in `derive.ts` — `definitionHash` is called from schema * validation and from a Convex mutation, neither of which can await. * * The mirror in `mcpjam-backend/convex/lib/scoreContract.ts` is proven equal by * the `__digests` block of `score-contract-parity-fixtures.json`, which pins * literal digests rather than recomputing an expectation on both sides. */ /** * Thrown when a value cannot be canonicalized. Deliberately loud: silently * coercing `NaN` to `null` (which is what `JSON.stringify` does) would let two * different configs share a digest. */ declare class CanonicalJsonError extends Error { constructor(message: string); } /** * RFC 8785-style canonical JSON. * * Rules, stated exactly so the Convex mirror can be read against them: * * 1. Object keys are sorted by UTF-16 code unit (plain `Array#sort`). * 2. Object properties whose value is `undefined` are DROPPED — this is what * makes an unresolved optional and an absent field digest identically. * 3. `undefined` inside an array becomes `null`, matching `JSON.stringify`: * dropping it would silently renumber positions. * 4. Numbers use the shortest round-trip form, with `-0` normalized to `0`. * Non-finite numbers throw. * 5. Strings use `JSON.stringify` escaping, which is already RFC 8785's. * 6. Anything else (function, symbol, bigint, class instance with `toJSON`) * throws rather than being coerced. */ declare function canonicalJson(value: unknown): string; /** Lowercase hex SHA-256 of a UTF-8 string. */ declare function sha256Hex(text: string): string; /** * The contract's one hashing entry point: canonicalize, then SHA-256. * * Returns bare lowercase hex (64 chars) with no algorithm prefix — the * narrowest possible surface for a hand-written mirror to reproduce. */ declare function canonicalDigest(value: unknown): string; /** * Zod mirrors of the evaluation contract. * * This module is browser-safe and intentionally has no node-only deps. * * These schemas are the authoring-side validator. Convex hand-mirrors them as * `v.*` validators (mutations can never be `'use node'`, and `v.*` arg * validators cannot be produced from zod), and the two are proven equal through * `sdk/tests/fixtures/score-contract-parity-fixtures.json` — the same mechanism * that keeps `predicateSchema` and `convex/lib/predicates.ts` honest. * * Every object is `.strict()`: Convex `v.object` REJECTS unknown fields, so a * permissive Zod schema here would accept a payload the backend refuses, and * the parity fixtures would certify two validators that disagree. * * The `superRefine` clauses are not decoration. `passed` is DERIVED * (`value >= passThreshold`), and a payload asserting otherwise is the exact * shape a tampered or malformed score set takes: a row claiming * `{value: 0.2, passThreshold: 0.7, passed: true}` would, on the * `resultSource === "reported"` path, turn a failing run into a passing one. * The derivation check is what makes that unrepresentable. */ declare const scoreStatusSchema: z.ZodEnum<{ error: "error"; skipped: "skipped"; scored: "scored"; not_applicable: "not_applicable"; }>; declare const scorerRoleSchema: z.ZodEnum<{ gating: "gating"; advisory: "advisory"; }>; declare const scorerErrorPolicySchema: z.ZodEnum<{ ignore: "ignore"; fail: "fail"; }>; declare const scorerIdSourceSchema: z.ZodEnum<{ explicit: "explicit"; generated: "generated"; platform: "platform"; }>; /** The authored definition: `onError`/`onSkipped` may still be unresolved. */ declare const scoreDefinitionSchema: z.ZodObject<{ onError: z.ZodOptional>; onSkipped: z.ZodOptional>; scorerId: z.ZodString; idSource: z.ZodEnum<{ explicit: "explicit"; generated: "generated"; platform: "platform"; }>; scorerVersion: z.ZodString; implementationHash: z.ZodString; label: z.ZodOptional; deterministic: z.ZodBoolean; passThreshold: z.ZodNumber; role: z.ZodEnum<{ gating: "gating"; advisory: "advisory"; }>; model: z.ZodOptional; scope: z.ZodOptional; promptIndex: z.ZodNumber; }, z.core.$strip>>; }, z.core.$strict>; /** * The hashing and wire form. Every semantic default is filled, which is what * makes an omitted `onError` and an explicitly-configured default digest * identically. */ declare const resolvedScoreDefinitionSchema: z.ZodObject<{ onError: z.ZodEnum<{ ignore: "ignore"; fail: "fail"; }>; onSkipped: z.ZodEnum<{ ignore: "ignore"; fail: "fail"; }>; scorerId: z.ZodString; idSource: z.ZodEnum<{ explicit: "explicit"; generated: "generated"; platform: "platform"; }>; scorerVersion: z.ZodString; implementationHash: z.ZodString; label: z.ZodOptional; deterministic: z.ZodBoolean; passThreshold: z.ZodNumber; role: z.ZodEnum<{ gating: "gating"; advisory: "advisory"; }>; model: z.ZodOptional; scope: z.ZodOptional; promptIndex: z.ZodNumber; }, z.core.$strip>>; }, z.core.$strict>; declare const scoreResultSchema: z.ZodObject<{ scorerId: z.ZodString; scorerVersion: z.ZodString; definitionHash: z.ZodString; status: z.ZodEnum<{ error: "error"; skipped: "skipped"; scored: "scored"; not_applicable: "not_applicable"; }>; value: z.ZodOptional; passThreshold: z.ZodNumber; passed: z.ZodOptional; rationale: z.ZodOptional; evidence: z.ZodOptional>; deterministic: z.ZodBoolean; model: z.ZodOptional; promptHash: z.ZodOptional; error: z.ZodOptional; scope: z.ZodOptional; promptIndex: z.ZodNumber; }, z.core.$strip>>; }, z.core.$strict>; declare const scoreResultArraySchema: z.ZodArray; value: z.ZodOptional; passThreshold: z.ZodNumber; passed: z.ZodOptional; rationale: z.ZodOptional; evidence: z.ZodOptional>; deterministic: z.ZodBoolean; model: z.ZodOptional; promptHash: z.ZodOptional; error: z.ZodOptional; scope: z.ZodOptional; promptIndex: z.ZodNumber; }, z.core.$strip>>; }, z.core.$strict>>; declare const evaluationConfigSnapshotSchema: z.ZodObject<{ hash: z.ZodString; definitions: z.ZodArray; onSkipped: z.ZodEnum<{ ignore: "ignore"; fail: "fail"; }>; scorerId: z.ZodString; idSource: z.ZodEnum<{ explicit: "explicit"; generated: "generated"; platform: "platform"; }>; scorerVersion: z.ZodString; implementationHash: z.ZodString; label: z.ZodOptional; deterministic: z.ZodBoolean; passThreshold: z.ZodNumber; role: z.ZodEnum<{ gating: "gating"; advisory: "advisory"; }>; model: z.ZodOptional; scope: z.ZodOptional; promptIndex: z.ZodNumber; }, z.core.$strip>>; }, z.core.$strict>>; }, z.core.$strict>; /** * Derivation for the evaluation contract — resolution, hashing, and the ONLY * sanctioned producer of a {@link ScoreResult}. * * This module is browser-safe and intentionally has no node-only deps. * * Every rule that could otherwise be re-implemented slightly differently by * each caller lives here exactly once: * * - defaults resolution (`onError`/`onSkipped` from `role`), * - what a definition hashes over, * - `passed = value >= passThreshold`, * - bounds/truncation on `rationale`, `evidence` and `error`, * - what a scorer that returned garbage becomes (an `error`, never a score). * * A scorer returns a {@link ScoreRawOutcome}; only `finalizeScoreResult` turns * one into a verdict. That is the structural reason a scorer cannot assert its * own `passed`. */ /** * Fill every semantic default. * * The gating defaults are fail-closed on purpose: a gating scorer that errors, * or that was expected to run and didn't, fails the iteration unless the author * explicitly opts out. An advisory scorer's statuses never gate, so both * policies default to `"ignore"` there — which also keeps the resolved shape * total (no field is ever absent) without inventing a third "n/a" policy. */ declare function resolveScoreDefinition(definition: ScoreDefinition): ResolvedScoreDefinition; /** Canonical-JSON + SHA-256 over one resolved definition. */ declare function definitionHash(definition: ResolvedScoreDefinition): string; /** * Canonical-JSON + SHA-256 over the whole resolved definition set. * * Definitions are sorted by their own canonical JSON before digesting, so the * hash is independent of authored order — a scorer moved up the list is not an * evaluation-config change. Stated precisely for the mirror: the digest is * SHA-256 of `"[" + sortedCanonicalPayloads.join(",") + "]"`, which is exactly * `canonicalDigest` of the sorted payload array. */ declare function evaluationConfigHash(definitions: ResolvedScoreDefinition[]): string; /** * Roll several per-case evaluation-config hashes into ONE run-level hash. * * A suite grades each case with its own definition set, but a run has a single * fingerprint. Digesting the sorted list of per-case hashes gives an * order-independent value that changes when any case's scorers change, and * also when a case is added or removed. * * Duplicates are deliberately KEPT: two cases sharing an identical scorer set * is a different configuration from one case with that set, and collapsing them * would hide a deleted case from the fingerprint. */ declare function aggregateEvaluationConfigHash(hashes: string[]): string; /** * Build the join table shipped with a run. Accepts authored definitions and * resolves them, so callers cannot accidentally hash an unresolved shape. * * `definitions` stays in authored order — dashboards read it top to bottom — * which is safe precisely because {@link evaluationConfigHash} sorts. */ declare function buildEvaluationConfigSnapshot(definitions: Array): EvaluationConfigSnapshot; /** * The verdict rule, in one place: a score passes when it reaches its threshold. * Never model-asserted, never scorer-asserted. */ declare function scorePassed(value: number, passThreshold: number): boolean; /** * A scorer that blew up. NEVER a low score: a crashed judge reporting `0` is * indistinguishable from a judge that ran and disagreed. Whether this fails the * iteration is the gate engine's decision, read off the definition's `onError`. */ declare function errorScoreResult(definition: ResolvedScoreDefinition, error: unknown, options?: { scope?: ScoreResult["scope"]; }): ScoreResult; /** * Expected work that did not happen. Gating scorers fail closed on this by * default: an unscored gate is not a passed gate. */ declare function skippedScoreResult(definition: ResolvedScoreDefinition, rationale?: string, options?: { scope?: ScoreResult["scope"]; }): ScoreResult; /** * The scorer does not apply to this iteration at all. Never gates, and never * enters an aggregation denominator — the distinction from `skipped` that keeps * "3 of 4 scorers passed" honest when the fourth was never in scope. */ declare function notApplicableScoreResult(definition: ResolvedScoreDefinition, rationale?: string, options?: { scope?: ScoreResult["scope"]; }): ScoreResult; /** * Turn one scorer's raw observation into a contract verdict. The ONLY * sanctioned producer of a {@link ScoreResult}. * * A `scored` outcome whose value is not a finite number in [0,1] finalizes to * `error`, not to a clamped score. Clamping `1.5` to `1` would turn a * malfunctioning judge into a passing gate, which is the same failure mode as * treating a crash as a `0` — just pointed the other way. */ declare function finalizeScoreResult(definition: ResolvedScoreDefinition, outcome: ScoreRawOutcome): ScoreResult; /** * Does the GATING evidence in this iteration's score rows say it passed? * * This is the arithmetic the score contract becomes AUTHORITATIVE with (grading * mode `enforce`): the inspector derives an iteration's `result` from it, and * the backend re-derives from the persisted rows and downgrades the iteration * if the two disagree. It lives here, in the contract's only sanctioned * derivation module, because two implementations of "did the gates hold" is * precisely how a hosted run and a CI gate end up disagreeing about the same * rows. `convex/lib/scoreContract.ts` in the backend hand-mirrors it, pinned by * the shared parity fixtures. * * ── The rule ──────────────────────────────────────────────────────────────── * * An iteration passes when EVERY gating definition resolved, and every gating * definition that resolved to a verdict passed. Stated as its two failure * modes, which are deliberately reported separately: * * - `disagreeingScorerIds` — a gating scorer RAN and said no. A real failure. * - `unresolvedScorerIds` — a gating scorer produced no usable verdict: no * row at all, or a row whose status its own `onError`/`onSkipped` policy * says must fail. ABSENCE of evidence, and it does not pass. Zero evidence * never passes — that is the whole reason the resolved defaults for a * gating definition are `fail`/`fail`. * * The separation is what lets one function serve both consumers. The authority * path (`enforce`) reads `passed` and treats an unresolved gate as a failure, * conservatively matching what the legacy boolean pipeline does with an * unscorable criterion. The SHADOW comparison reads `disagreeingScorerIds` * alone (see `shadowVerdictFromScores` in the inspector's `score-rows.ts`), so * an honest error row cannot manufacture a mismatch out of a criterion nobody * could score. * * ── What cannot influence it ──────────────────────────────────────────────── * * - **Advisory rows.** Only `role: "gating"` definitions are iterated at all, * which is what makes the judge structurally incapable of gating rather * than conventionally excluded from it. * - **`not_applicable` rows.** They are excluded from every denominator — * that is what the status means — so a definition whose only rows are * `not_applicable` neither fails nor counts as missing. * - **Rows that do not join.** The join is by `definitionHash`, like every * other contract consumer: matching on `scorerId` would grade a row against * whichever definition landed last when a merged iteration carries the same * id under two hashes. A row whose hash matches no definition is already * quarantined at ingest (`validateScorePayload`), so accepted rows always * join. */ declare function allGatingScorersPassed(scores: readonly ScoreResult[], config: EvaluationConfigSnapshot): { passed: boolean; disagreeingScorerIds: string[]; unresolvedScorerIds: string[]; }; /** * The user-facing words for the eval contract's closed vocabularies. * * This module is browser-safe and intentionally has no node-only deps. * * Every enum this initiative pins is a WIRE spelling — `userValue`, * `argumentMismatch`, `evaluatorErrorRateAboveMaximum`. Those are correct on * the wire and wrong in front of a human, and until now each surface invented * its own rendering: the CLI printed the raw enum, the HTML report printed the * raw enum, and a future UI would have invented a third spelling. One map per * vocabulary, in one place, is what makes "first failed stage: User value" * mean the same thing in a terminal, in a CI artifact and in a browser. * * ── Why these are `satisfies Record` ─────────────────────────── * * Every map below is total over its vocabulary and says so to the compiler. * Adding a stage reason, a failure category or a verdict reason to the contract * therefore breaks THIS FILE until somebody writes the words a human reads — * which is the point. The alternative, a lookup with a `?? value` fallback, * fails silently by printing the new enum member raw, and the surface that * looks most correct (it rendered something!) is the one nobody notices is * wrong. * * ── What is deliberately NOT here ──────────────────────────────────────────── * * No sentence here diagnoses anything. A first failed stage is where the chain * stopped, and a failure category is the bucket a run is grouped under; neither * is a claim about WHY it stopped, and phrasing that suggests otherwise is how * an operator ends up "fixing" the wrong system. */ /** * The six chain stages, in words. * * `userValue` is the one that matters: it is the stage a reader is most likely * to see (it is last, so it is where a mechanically-perfect run still fails) * and it is the one whose wire spelling reads worst. */ declare const USER_VALUE_STAGE_LABELS: Readonly<{ connection: string; discovery: string; selection: string; call: string; response: string; userValue: string; }>; /** * What a stage did. * * The three non-verdicts stay three different sentences, exactly as * `STAGE_STATES` insists: "we did not check", "it does not apply" and "it never * ran" are different facts, and one shared word for them is how "we never * checked" gets read as "it passed". */ declare const STAGE_STATE_LABELS: Readonly<{ passed: string; failed: string; notReached: string; notMeasured: string; notApplicable: string; }>; /** The coarse bucket a non-passing run is grouped under. */ declare const FAILURE_CATEGORY_LABELS: Readonly<{ setup: string; metadata: string; selection: string; arguments: string; serverData: string; userValue: string; evaluator: string; }>; /** * Why a stage landed where it did. * * Written as fragments that complete "…because ", so a renderer can * splice one into a line without a per-reason special case. */ declare const STAGE_REASON_LABELS: Readonly<{ noSpanChannel: string; noEvidenceCaptured: string; matchVerdictUnavailable: string; traceAbsent: string; executorEmitsNoSpans: string; blockedByPolicy: string; evaluatorError: string; setupAborted: string; connectFailed: string; toolsListFailed: string; egressUnverified: string; lifecycleStopped: string; notAuthored: string; earlierStageFailed: string; missingToolCall: string; unexpectedToolCall: string; argumentMismatch: string; toolError: string; protocolError: string; renderFailed: string; predicateFailed: string; observed: string; impliedByLaterEvidence: string; judgeObserved: string; judgePartial: string; judgeFailed: string; judgePending: string; judgeNotRequested: string; }>; /** * Why a v2 run's verdict is what it is. * * These are the audit trail an `inconclusive` run is explained by, and they are * the single most useful thing to put in front of someone staring at a run that * neither passed nor failed. Phrased as statements of what was measured, never * as blame. */ declare const EVAL_VERDICT_DECISION_REASON_LABELS: Readonly<{ configuredTrialsNotAttempted: string; noGradeableTrials: string; eligibleTrialsBelowMinimum: string; completionRateBelowMinimum: string; completionRateNotMeasured: string; evaluatorErrorRateAboveMaximum: string; evaluatorErrorRateNotMeasured: string; caseHasNoEligibleTrials: string; casePassRateMetThreshold: string; casePassRateBelowThreshold: string; allMeasuredCasesMetThreshold: string; }>; /** * The operator action for one failure category. * * Relocated here from `src/eval-decision-summary.ts`, which still re-exports it * under its published name. One action per category, and the category is the * only input: an action keyed on anything finer would be a diagnosis, and this * contract does not diagnose. */ declare const NEXT_ACTION_BY_FAILURE_CATEGORY: Readonly<{ setup: string; metadata: string; selection: string; arguments: string; serverData: string; userValue: string; evaluator: string; }>; /** * The action when no failure category was established. * * Deliberately says to go and look rather than naming a system: with no * category there is no evidence about which one is involved, and a confident * suggestion here would be invention. */ declare const DECISION_SUMMARY_FALLBACK_NEXT_ACTION = "inspect the case trace; no failure category was recorded"; /** Every vocabulary this module renders, for tests that assert totality. */ declare const DECISION_LABEL_VOCABULARIES: Readonly<{ stages: readonly ["connection", "discovery", "selection", "call", "response", "userValue"]; stageStates: readonly ["passed", "failed", "notReached", "notMeasured", "notApplicable"]; failureCategories: readonly ["setup", "metadata", "selection", "arguments", "serverData", "userValue", "evaluator"]; stageReasons: readonly ["noSpanChannel", "noEvidenceCaptured", "matchVerdictUnavailable", "traceAbsent", "executorEmitsNoSpans", "blockedByPolicy", "evaluatorError", "setupAborted", "connectFailed", "toolsListFailed", "egressUnverified", "lifecycleStopped", "notAuthored", "earlierStageFailed", "missingToolCall", "unexpectedToolCall", "argumentMismatch", "toolError", "protocolError", "renderFailed", "predicateFailed", "observed", "impliedByLaterEvidence", "judgeObserved", "judgePartial", "judgeFailed", "judgePending", "judgeNotRequested"]; verdictDecisionReasons: readonly ["configuredTrialsNotAttempted", "noGradeableTrials", "eligibleTrialsBelowMinimum", "completionRateBelowMinimum", "completionRateNotMeasured", "evaluatorErrorRateAboveMaximum", "evaluatorErrorRateNotMeasured", "caseHasNoEligibleTrials", "casePassRateMetThreshold", "casePassRateBelowThreshold", "allMeasuredCasesMetThreshold"]; }>; export { CanonicalJsonError as C, DECISION_LABEL_VOCABULARIES as D, EVAL_VERDICT_DECISION_REASON_LABELS as E, FAILURE_CATEGORY_LABELS as F, NEXT_ACTION_BY_FAILURE_CATEGORY as N, STAGE_REASON_LABELS as S, USER_VALUE_STAGE_LABELS as U, DECISION_SUMMARY_FALLBACK_NEXT_ACTION as a, STAGE_STATE_LABELS as b, aggregateEvaluationConfigHash as c, allGatingScorersPassed as d, buildEvaluationConfigSnapshot as e, canonicalDigest as f, canonicalJson as g, definitionHash as h, errorScoreResult as i, evaluationConfigHash as j, evaluationConfigSnapshotSchema as k, finalizeScoreResult as l, resolvedScoreDefinitionSchema as m, notApplicableScoreResult as n, scorePassed as o, scoreResultArraySchema as p, scoreResultSchema as q, resolveScoreDefinition as r, scoreDefinitionSchema as s, scoreStatusSchema as t, scorerErrorPolicySchema as u, scorerIdSourceSchema as v, scorerRoleSchema as w, sha256Hex as x, skippedScoreResult as y };