/** * evaluation pipeline orchestrator —— 单次 evaluation run 的「执行+收尾」编排。 * * 主入口 `executeEvaluationPipeline` 把以下几段串成线性时序: * 1. `initializeEvaluationRunState` 建 run / job 元数据并写 jobStore * 2. resolve judge models + executor map(必须先于 preflight,见函数内注释) * 3. `preflight` + `preflightAllJudges` LLM 连通性 * 4. `emitPowerWarnings` + `emitIsolationWarnings` 结构性 / 隔离性预警 * 5. `executeTasks` 真正跑用例 * 6. `finalizeSuccessfulRun` + `finalizeEvaluationReport` 收尾 + 后置分析 * 7. `persistReport` + `persistSuccessfulJob` 落盘 * 失败路径走 `persistFailedJob`, finally 关 MCP server。 * * 4 块被拆出去的 helper 都在 `./evaluation-pipeline/` 目录下,与本文件 1:1 协作: * - run-state.ts: EvaluationRunState 生命周期(init / finalizeSuccess / persistSuccess / persistFailed) * - test-set-hash.ts: 测试集水印 hash(spec §7.1) * - preflight-warnings.ts: power + isolation 预警 * - report-finalize.ts: report 后置分析装配 * * 兼容 re-export: `buildPowerWarnings` / `buildIsolationWarnings` / * `_computeTestSetHashForTest` 保留在本模块的 export surface,避免 test 与 * `run-evaluation.ts:275` 动态 import 路径改动。 */ import type { DependencyRequirements } from '../eval-core/dependency-checker.js'; import type { Artifact, ExecutorFn, JobStore, ProgressCallback, Report, Sample, Task } from '../types/index.js'; import type { Lang } from '../types/shared.js'; export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js'; export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js'; export interface EvaluationPipelineOptions { samplesPath: string; /** Sample bundle 根目录。loadSamples 输出,缺省时 fallback dirname(samplesPath)。 */ samplesBaseDir?: string; /** Sample bundle 内合并的源文件绝对路径列表。目录模式下用于 computeTestSetHash 遍历。 */ samplesSourceFiles?: string[]; skillDir: string; samples: Sample[]; tasks: Task[]; artifacts: Artifact[]; model: string; judgeModel: string; noJudge: boolean; executorName: string; judgeExecutorName: string; executor: ExecutorFn; judgeExecutor: ExecutorFn; outputDir?: string | null; project?: string; owner?: string; tags?: string[]; concurrency?: number; timeoutMs?: number; noCache?: boolean; jobStore?: JobStore | null; persistJob?: boolean; onProgress?: ProgressCallback | null; skipConnectivity?: boolean; verbose?: boolean; retry?: number; existingResults?: Record>; requires?: DependencyRequirements; layeredStats?: boolean; /** 透传到 meta.request.repeat */ repeat?: number; /** 透传到 meta.request.holdoutRatio;> 0 时 report-finalize 算 train/holdout 子集综合分。 */ holdoutRatio?: number; /** 透传到 meta.request.batch */ batch?: boolean; /** 透传到 meta.request.judgeRepeat 与 grade(),每条 sample × dimension judge N 次 */ judgeRepeat?: number; /** Multi-judge ensemble configs (≥ 2 entries triggers ensemble mode). */ judgeModels?: import('../types/index.js').JudgeConfig[]; /** --bootstrap. */ bootstrap?: boolean; /** --bootstrap-samples N. Default 1000. */ bootstrapSamples?: number; /** v0.21 length-debias toggle. Default true; --no-debias-length flips to false. */ lengthDebias?: boolean; /** hard budget caps. */ budget?: import('../types/index.js').EvalBudget; /** strict-baseline default state (only used to decide whether to emit * isolation-disabled pre-flight warnings). True/undefined = default behavior * (no warning); false = user explicitly disabled, warn if ~/.claude/skills/ has content. */ strictBaseline?: boolean; /** Explicit persisted run id. Used by batch workflows that need stable child ids. */ runId?: string; /** CLI output language for warnings emitted by the pipeline. */ lang?: Lang; /** Reasoning effort 透传到 executor。默认 'low'(由 RunConfig 兜底)。 */ effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max'; /** 关闭 diagnostic。Default false。 */ noDiagnostic?: boolean; } type VariantResult = import('../types/index.js').VariantResult; export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{ report: Report; filePath: string | null; }>;