{"version":3,"file":"eval-runner.mjs","names":["judgeScorer"],"sources":["../../../../../../../ai/src/eval/eval-runner.ts"],"sourcesContent":["import type { AgentContract } from \"../contracts/agent/agent.contract\";\nimport type { AgentExecuteOptions } from \"../contracts/agent/agent-options.type\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalOptions,\n  EvalReport,\n  EvalScore,\n  EvalScorer,\n  EvalScorerContext,\n} from \"../contracts/agent/eval.type\";\nimport type { EvalCase as EvalCaseType } from \"../contracts/agent/eval.type\";\nimport { AgentExecutionError } from \"../errors\";\nimport { log } from \"@warlock.js/logger\";\nimport { judge as judgeScorer } from \"./judge-scorer\";\nimport { diff } from \"./regression\";\n\n/**\n * Narrow `EvalOptions.cases` to the underlying `EvalCase[]`. A\n * `DatasetContract` is identified structurally by its `cases` property\n * (an array carried alongside `name` / `filter` / `shard`); a raw\n * `EvalCase[]` is used as-is.\n */\nfunction resolveCases<TOutput>(\n  cases: EvalOptions<TOutput>[\"cases\"],\n): EvalCaseType<TOutput>[] {\n  if (Array.isArray(cases)) {\n    return cases;\n  }\n\n  return cases.cases;\n}\n\nconst LOG_MODULE = \"ai.eval\";\nconst DEFAULT_PASS_THRESHOLD = 0.5;\n\n/**\n * Resolve the scorer list for a single case. Precedence: the case's\n * own `scorers` → the suite `scorers` → a synthesized judge scorer\n * when `judge` is configured. Throws an authoring-time\n * `AgentExecutionError` when a case can resolve none — an eval suite\n * with no way to score a case is a config bug worth surfacing at the\n * call site, not a silent pass.\n */\nfunction resolveScorers<TOutput>(\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): EvalScorer<TOutput>[] {\n  if (evalCase.scorers && evalCase.scorers.length > 0) {\n    return evalCase.scorers;\n  }\n\n  if (options.scorers && options.scorers.length > 0) {\n    return options.scorers;\n  }\n\n  if (options.judge) {\n    return [judgeScorer<TOutput>(options.judge, passThreshold)];\n  }\n\n  throw new AgentExecutionError(\n    `eval case \"${evalCase.name}\" has no scorer — supply per-case \"scorers\", suite \"scorers\", or a \"judge\"`,\n    { context: { authoring: true, case: evalCase.name } },\n  );\n}\n\n/**\n * Decide a single scorer verdict's pass/fail. Honors an explicit\n * `passed` from the scorer; otherwise derives it from\n * `score >= passThreshold`.\n */\nfunction isScorePassing(score: EvalScore, passThreshold: number): boolean {\n  if (typeof score.passed === \"boolean\") {\n    return score.passed;\n  }\n\n  return score.score >= passThreshold;\n}\n\n/**\n * Merge suite-level execute options with the case's own override.\n * Per-case wins on conflict (shallow merge).\n */\nfunction mergeOptions<TOutput>(\n  suite: AgentExecuteOptions<TOutput> | undefined,\n  perCase: AgentExecuteOptions<TOutput> | undefined,\n): AgentExecuteOptions<TOutput> | undefined {\n  if (!suite) return perCase;\n  if (!perCase) return suite;\n  return { ...suite, ...perCase };\n}\n\n/**\n * Run one case end-to-end: execute the agent, run every resolved\n * scorer, aggregate into an {@link EvalCaseResult}. A case passes only\n * when the agent did not error AND every scorer passed.\n */\nasync function runCase<TOutput>(\n  agent: AgentContract<TOutput>,\n  evalCase: EvalCase<TOutput>,\n  options: EvalOptions<TOutput>,\n  passThreshold: number,\n): Promise<EvalCaseResult<TOutput>> {\n  const scorers = resolveScorers(evalCase, options, passThreshold);\n  const executeOptions = mergeOptions(options.executeOptions, evalCase.options);\n\n  const start = performance.now();\n  const result = await agent.execute(evalCase.input, executeOptions);\n  const duration = performance.now() - start;\n\n  const context: EvalScorerContext<TOutput> = {\n    case: evalCase,\n    result,\n    output: result.data,\n    text: result.text,\n  };\n\n  const scores: EvalScore[] = [];\n\n  for (const scorer of scorers) {\n    scores.push(await scorer(context));\n  }\n\n  const meanScore =\n    scores.length > 0 ? scores.reduce((sum, score) => sum + score.score, 0) / scores.length : 0;\n\n  const allScorersPassed = scores.every((score) => isScorePassing(score, passThreshold));\n  const passed = result.error === undefined && allScorersPassed;\n\n  return {\n    case: evalCase,\n    result,\n    scores,\n    score: meanScore,\n    passed,\n    duration,\n  };\n}\n\n/**\n * Core implementation of `agent.eval`. Runs every case sequentially\n * (cases share the agent and may carry side effects — ordering must be\n * deterministic), scores each, fires `onFailure` for failed cases, and\n * assembles the aggregate {@link EvalReport}.\n *\n * Never throws on a case-level failure; the only throw is the\n * authoring-time \"no scorer\" guard from {@link resolveScorers}.\n */\nexport async function runEval<TOutput>(\n  agent: AgentContract<TOutput>,\n  options: EvalOptions<TOutput>,\n): Promise<EvalReport<TOutput>> {\n  const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;\n  const start = performance.now();\n\n  const suiteCases = resolveCases(options.cases);\n  const cases: EvalCaseResult<TOutput>[] = [];\n\n  for (const evalCase of suiteCases) {\n    const caseResult = await runCase(agent, evalCase, options, passThreshold);\n\n    cases.push(caseResult);\n\n    if (!caseResult.passed && options.onFailure) {\n      try {\n        await options.onFailure(caseResult);\n      } catch (error) {\n        log.warn(LOG_MODULE, \"onFailure.hook.error\", \"eval onFailure handler threw\", {\n          agent: agent.name,\n          case: evalCase.name,\n          error: error instanceof Error ? error.message : String(error),\n        });\n      }\n    }\n  }\n\n  const passedCount = cases.filter((entry) => entry.passed).length;\n  const total = cases.length;\n  const meanScore =\n    total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;\n\n  const report: EvalReport<TOutput> = {\n    agentName: agent.name,\n    total,\n    passedCount,\n    failedCount: total - passedCount,\n    passRate: total > 0 ? passedCount / total : 0,\n    meanScore,\n    passed: total > 0 && passedCount === total,\n    cases,\n    duration: performance.now() - start,\n  };\n\n  if (options.baseline) {\n    report.regression = diff(report, options.baseline, options.tolerance);\n  }\n\n  return report;\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,SAAS,aACP,OACyB;CACzB,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO;CAGT,OAAO,MAAM;AACf;AAEA,MAAM,aAAa;AACnB,MAAM,yBAAyB;;;;;;;;;AAU/B,SAAS,eACP,UACA,SACA,eACuB;CACvB,IAAI,SAAS,WAAW,SAAS,QAAQ,SAAS,GAChD,OAAO,SAAS;CAGlB,IAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAC9C,OAAO,QAAQ;CAGjB,IAAI,QAAQ,OACV,OAAO,CAACA,MAAqB,QAAQ,OAAO,aAAa,CAAC;CAG5D,MAAM,IAAI,oBACR,cAAc,SAAS,KAAK,6EAC5B,EAAE,SAAS;EAAE,WAAW;EAAM,MAAM,SAAS;CAAK,EAAE,CACtD;AACF;;;;;;AAOA,SAAS,eAAe,OAAkB,eAAgC;CACxE,IAAI,OAAO,MAAM,WAAW,WAC1B,OAAO,MAAM;CAGf,OAAO,MAAM,SAAS;AACxB;;;;;AAMA,SAAS,aACP,OACA,SAC0C;CAC1C,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,CAAC,SAAS,OAAO;CACrB,OAAO;EAAE,GAAG;EAAO,GAAG;CAAQ;AAChC;;;;;;AAOA,eAAe,QACb,OACA,UACA,SACA,eACkC;CAClC,MAAM,UAAU,eAAe,UAAU,SAAS,aAAa;CAC/D,MAAM,iBAAiB,aAAa,QAAQ,gBAAgB,SAAS,OAAO;CAE5E,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,SAAS,MAAM,MAAM,QAAQ,SAAS,OAAO,cAAc;CACjE,MAAM,WAAW,YAAY,IAAI,IAAI;CAErC,MAAM,UAAsC;EAC1C,MAAM;EACN;EACA,QAAQ,OAAO;EACf,MAAM,OAAO;CACf;CAEA,MAAM,SAAsB,CAAC;CAE7B,KAAK,MAAM,UAAU,SACnB,OAAO,KAAK,MAAM,OAAO,OAAO,CAAC;CAGnC,MAAM,YACJ,OAAO,SAAS,IAAI,OAAO,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,OAAO,SAAS;CAE5F,MAAM,mBAAmB,OAAO,OAAO,UAAU,eAAe,OAAO,aAAa,CAAC;CAGrF,OAAO;EACL,MAAM;EACN;EACA;EACA,OAAO;EACP,QAPa,OAAO,UAAU,UAAa;EAQ3C;CACF;AACF;;;;;;;;;;AAWA,eAAsB,QACpB,OACA,SAC8B;CAC9B,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,QAAQ,YAAY,IAAI;CAE9B,MAAM,aAAa,aAAa,QAAQ,KAAK;CAC7C,MAAM,QAAmC,CAAC;CAE1C,KAAK,MAAM,YAAY,YAAY;EACjC,MAAM,aAAa,MAAM,QAAQ,OAAO,UAAU,SAAS,aAAa;EAExE,MAAM,KAAK,UAAU;EAErB,IAAI,CAAC,WAAW,UAAU,QAAQ,WAChC,IAAI;GACF,MAAM,QAAQ,UAAU,UAAU;EACpC,SAAS,OAAO;GACd,IAAI,KAAK,YAAY,wBAAwB,gCAAgC;IAC3E,OAAO,MAAM;IACb,MAAM,SAAS;IACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D,CAAC;EACH;CAEJ;CAEA,MAAM,cAAc,MAAM,QAAQ,UAAU,MAAM,MAAM,CAAC,CAAC;CAC1D,MAAM,QAAQ,MAAM;CACpB,MAAM,YACJ,QAAQ,IAAI,MAAM,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,CAAC,IAAI,QAAQ;CAE3E,MAAM,SAA8B;EAClC,WAAW,MAAM;EACjB;EACA;EACA,aAAa,QAAQ;EACrB,UAAU,QAAQ,IAAI,cAAc,QAAQ;EAC5C;EACA,QAAQ,QAAQ,KAAK,gBAAgB;EACrC;EACA,UAAU,YAAY,IAAI,IAAI;CAChC;CAEA,IAAI,QAAQ,UACV,OAAO,aAAa,KAAK,QAAQ,QAAQ,UAAU,QAAQ,SAAS;CAGtE,OAAO;AACT"}