/** * Eval-campaign — the app-shell's curated surface for a product's * self-improvement loop, NOT a reimplementation. * * The loop ENGINE lives in `@tangle-network/agent-eval` (a peer dependency): * `selfImprove` already owns execution, scoring, data separation, release * decisions, durable provenance, and hosted ingest. Candidate search is always * explicit: pass an official optimization `method`, or pass a caller-owned * `SurfaceProposer`. * * This module adds the one piece `selfImprove` does not own and which every * multi-model product re-hand-rolls — the ensemble judge: * * {@link buildEnsembleJudge} — turn a per-rubric `scoreOne` into a * `JudgeConfig` that fans out N uncorrelated judge calls and reduces them via * the substrate's `aggregateJudgeVerdicts` (survivor-mean, inter-rater spread, * fail-loud on all-failed). A product writes its rubric + one judge call; the * fan-out, partial-failure handling, and composite are the scaffold's. * * Everything else is a curated re-export so a product has ONE eval import: * `selfImprove` + release policies + optimization methods + their types. See * `.claude/skills/eval-campaign/SKILL.md` for the wiring contract. */ import { type JudgeVerdict } from '@tangle-network/agent-eval'; import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'; /** Config for {@link buildEnsembleJudge}. `D` = the rubric's dimension union. */ export interface EnsembleJudgeConfig { /** Judge name — appears in traces and scorecards. */ name: string; /** Stable-ordered rubric dimensions. Drives the `JudgeDimension` list AND the * reducer keys, so a judge that omits a dimension scores it 0 (never silently * dropped). */ rubric: readonly D[]; /** * Score ONE artifact on the rubric → a raw per-dimension verdict. Called * `judgeReps` times per artifact; vary the model by `rep` for an uncorrelated * ensemble (judges that share a base model share its bias). Return * `{ model, perDimension: null }` to record a judge failure WITHOUT killing * the ensemble; throw only on an unrecoverable error (the whole rep is then * treated as a failed judge). */ scoreOne: (input: { artifact: TArtifact; scenario: TScenario; signal: AbortSignal; rep: number; }) => Promise>; /** Independent judge calls per artifact, reduced by `aggregateJudgeVerdicts`. * Default 1. Raise (with model variety in `scoreOne`) for inter-rater bands. */ judgeReps?: number; /** Per-dimension composite weights. Default: uniform over `rubric`. A partial * map selects-and-weights exactly the named dimensions. */ weights?: Partial>; /** Optional human-readable dimension descriptions. Default: the key itself. */ describe?: (dim: D) => string; } /** * Build a `JudgeConfig` whose `score()` fans out `judgeReps` independent * `scoreOne` calls and reduces them with the substrate's * `aggregateJudgeVerdicts`. A single judge call failing does NOT fail the cell * (it is recorded and dropped); only ALL judges failing throws — which the * campaign records as a failed cell, never a silent zero. * * Pass the result straight to `selfImprove({ judge })` (or `runCampaign`). */ export declare function buildEnsembleJudge(cfg: EnsembleJudgeConfig): JudgeConfig; export { trustVerdicts, type TrustItem, type TrustThresholds, type TrustVerdict, } from './trust-gate'; export { aggregateJudgeVerdicts } from '@tangle-network/agent-eval'; export type { EnsembleAggregate, JudgeVerdict, RunRecord, } from '@tangle-network/agent-eval'; export { compareOptimizationMethods, defaultProductionGate, externalTextOptimizationMethod, gepaOptimizationMethod, paretoSignificanceGate, runCampaign, skillOptOptimizationMethod, } from '@tangle-network/agent-eval/campaign'; export type { CampaignResult, CompareOptimizationMethodsOptions, DispatchContext, ExternalTextOptimizationMethodConfig, Gate, GepaOptimizationMethodConfig, JudgeConfig, JudgeDimension, JudgeScore, LabeledScenarioStore, MutableSurface, OptimizationMethod, OptimizationMethodResult, Scenario, SkillOptOptimizationMethodConfig, SurfaceProposer, } from '@tangle-network/agent-eval/campaign'; export { selfImprove } from '@tangle-network/agent-eval/contract'; export type { SelfImproveBudget, SelfImproveOptions, SelfImproveResult, } from '@tangle-network/agent-eval/contract';