import { z } from 'zod'; /** * Canonical schema for the grader agent's `grading.json` output. * * SOURCE OF TRUTH: skill-creator's `references/schemas.md` (the `grading.json` * section). vat consumes that exact shape; this module is the machine-checkable * encoding of it (skill-creator ships prose + an example, but no JSON Schema). * The published JSON Schema is {@link GradingReportJsonSchema}. * * SHAPE: a SINGLE flat JSON object with two load-bearing top-level fields — * * { * "expectations": [ { "text": string, "passed": boolean, "evidence"?: string }, ... ], * "summary": { "passed": number, "total": number, "failed"?: number, "pass_rate"?: number } * } * * `expectations` holds ONE entry per graded expectation across ALL evals — it is * NOT grouped per-eval and is NEVER wrapped in an `evals` array. A per-eval * nested shape (`{ evals: [ { expectations, summary } ] }`) is a contract * violation and is rejected loudly (see grading-adapter.ts); tolerating it would * push malformed data downstream and create confusion. * * LIBERAL ON EXTRAS (Postel): the grader legitimately emits additional documented * sections — `execution_metrics`, `timing`, `claims`, `user_notes_summary`, * `eval_feedback` — plus viewer URLs and other adornments. We `.passthrough()` * those: validate the two fields we depend on, carry the rest untouched. Extra * fields are NOT "bad JSON"; a wrong top-level STRUCTURE is. */ /** One graded expectation. `evidence` is recommended but not load-bearing for vat. */ export declare const GradedExpectationSchema: z.ZodObject<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; export type GradedExpectation = z.infer; /** * Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. * Counts are non-negative integers (a float or negative count is a grader bug), * and `passed` can never exceed `total`. */ export declare const GradingSummarySchema: z.ZodEffects; pass_rate: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; export type GradingSummary = z.infer; /** * The full grading.json contract. Required: top-level `expectations[]` and * `summary`. Everything else passes through untouched (forward-compatible with * skill-creator additions). */ export declare const GradingReportSchema: z.ZodObject<{ expectations: z.ZodArray; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, "many">; summary: z.ZodEffects; pass_rate: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; /** * Per-run integrity nonce each grader is told to copy verbatim from its * prompt into its fragment (see grader-prompt.ts). Optional in the schema (a * grading.json validated by external tooling need not carry it), but the * harness REQUIRES every merged fragment's nonce to match the run's secret * nonce before trusting the verdict — this is how a forged fragment written * by untrusted skill code is rejected. */ runNonce: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ expectations: z.ZodArray; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, "many">; summary: z.ZodEffects; pass_rate: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; /** * Per-run integrity nonce each grader is told to copy verbatim from its * prompt into its fragment (see grader-prompt.ts). Optional in the schema (a * grading.json validated by external tooling need not carry it), but the * harness REQUIRES every merged fragment's nonce to match the run's secret * nonce before trusting the verdict — this is how a forged fragment written * by untrusted skill code is rejected. */ runNonce: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ expectations: z.ZodArray; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ text: z.ZodString; passed: z.ZodBoolean; evidence: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, "many">; summary: z.ZodEffects; pass_rate: z.ZodOptional; }, "passthrough", z.ZodTypeAny, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">, z.objectInputType<{ passed: z.ZodNumber; total: z.ZodNumber; failed: z.ZodOptional; pass_rate: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; /** * Per-run integrity nonce each grader is told to copy verbatim from its * prompt into its fragment (see grader-prompt.ts). Optional in the schema (a * grading.json validated by external tooling need not carry it), but the * harness REQUIRES every merged fragment's nonce to match the run's secret * nonce before trusting the verdict — this is how a forged fragment written * by untrusted skill code is rejected. */ runNonce: z.ZodOptional; }, z.ZodTypeAny, "passthrough">>; export type GradingReport = z.infer; /** * Published JSON Schema for grading.json — generated from {@link GradingReportSchema} * so the two never drift. Importable by external tooling that wants to validate a * grading.json without depending on Zod. Documented in * docs/skill-test-grading-schema.md. */ export declare const GradingReportJsonSchema: import("zod-to-json-schema").JsonSchema7Type & { $schema?: string | undefined; definitions?: { [key: string]: import("zod-to-json-schema").JsonSchema7Type; } | undefined; }; //# sourceMappingURL=grading-schema.d.ts.map