/** * oclif typed flags → `RunConfig` 装配。 * * 主体 `parseRunConfig` 是「CLI > eval.yaml > 硬编码 default」精度链的统一落点 —— * 把 oclif strict 模式接受过的 flag values + 可选 eval.yaml 合并成 eval 子命令运行 * 所需的完整 `RunConfig`,后续 `executeEvaluationPipeline` 与 doctor / batch 都从这 * 里读字段。 * * 3 个 sub-routine 拆到同级 parse-run-config/ 目录,主文件保留它们的 re-export * 让外部 import surface(`parseJudgeModelsArg` / `parseJudgeModelsArgOrExit`) * 不动: * - judge-models.ts: --judge-models 字符串解析 * - samples-discovery.ts: 未传 --samples 时的路径发现 * - variant-resolution.ts: --control / --treatment / eval.yaml.variants 三态合并 * * 主体 parseRunConfig 自身保留约 100 行的 field-default fanout —— 每个字段一行 * `cli ?? evalConfig ?? default`,刻意不再细拆,否则只是把一长串 ?? 散到多文件, * 反而失去「精度链一目了然」的可读性。 */ import type { EvalConfig, VariantSpec, JudgeConfig, EvalBudget, ProgressCallback } from '../../types/index.js'; import { type RuntimeResolutionOptions } from './runtime-defaults.js'; export { parseJudgeModelsArg, parseJudgeModelsArgOrExit } from './parse-run-config/judge-models.js'; export interface RunConfig { samplesPath: string; skillDir: string; variantSpecs: VariantSpec[]; model: string; outputDir: string; noJudge: boolean | undefined; noCache: boolean | undefined; dryRun: boolean | undefined; concurrency: number; timeoutMs: number; executorName: string; /** 跳过 LLM 模型连通性检测。仅当 --resume 报告通过完整契约校验时自动 true。 */ skipConnectivity: boolean | undefined; /** 跳过 doctor 健康检查门禁(--skip-doctor)。escape hatch — 默认 false。 * 开启后 doctor 整段不跑(节省静态检查时间);doctor 失败也不再阻断 eval。 * 典型场景:依赖在评测环境中通过 mock / stub 提供,doctor 的物理路径检查 * 会误报。开启意味着用户接受 garbage-in 风险,自己负责依赖正确性。 */ skipDoctor: boolean | undefined; /** 用户语言, 透传给 doctor 报告渲染。 */ lang: 'zh' | 'en' | undefined; mcpConfig: string | undefined; verbose: boolean | undefined; retry?: number; resume?: string; layeredStats?: boolean; /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice; report-finalize * computes train vs holdout composite (report.analysis.holdout) for the overfitting gate. */ holdoutRatio?: number; /** --judge-repeat N. Calls LLM judge N times per (sample × dimension). Default 1. */ judgeRepeat?: number; /** Unified judge config. Always non-empty; 1 entry = single judge, ≥ 2 = ensemble. * Defaults to the selected runtime's judge model. */ judgeModels: JudgeConfig[]; /** --bootstrap. Adds bootstrap CI to summary (per-variant mean + pairwise diff). */ bootstrap?: boolean; /** --bootstrap-samples N. Bootstrap resamples count, default 1000. */ bootstrapSamples?: number; /** length-debias toggle. Default true; --no-debias-length sets false. */ lengthDebias?: boolean; /** hard budget caps from CLI or config. */ budget?: EvalBudget; /** Skill isolation default for baseline-kind variants. Default true. * CLI flag --no-strict-baseline disables strict isolation. */ strictBaseline?: boolean; /** Per-variant allowedSkills override extracted from eval.yaml. Always wins * over strictBaseline default. Keyed by variant name. */ variantAllowedSkills?: Record; /** Reasoning effort for the executor LLM(被评测的模型,不是 judge)。 * Default 'low' — sonnet 默认会做大量扩展思考(13K thinking tokens / 单次), * 对结构化任务是浪费。低 effort 大幅省时间/成本但可能损失复杂推理质量, * 跨 effort 的报告不能严格比较。`undefined` 走 claude CLI / SDK 自身默认。 */ effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max'; /** 关闭 diagnostic LLM call。Default false(总是给 failed sample 跑诊断)。 * 跟 noJudge 完全独立 — judge 答打分,diagnostic 答怎么改。 */ noDiagnostic?: boolean; onProgress?: ProgressCallback | null; } export interface ParseRunConfigResult { values: Record; config: RunConfig; /** Loaded eval.yaml when --config was provided. `commands/eval-runner.ts` uses it to apply * CLI > eval.yaml > default fallback for fields not propagated by parseRunConfig * (e.g. repeat / judgeRepeat / bootstrap — handled in `commands/eval-runner.ts` for input validation). */ evalConfig: EvalConfig | null; } /** * 接 typed flags(来自 oclif Command.parse() 输出)。oclif strict 模式已经在 * 上游对未知 flag 拦截 exit 2,这里不再 parseArgs。eval-runner 等业务 caller * 把 oclif flags 当 values 喂进来。 */ export declare function parseRunConfig(values: Record, runtimeOptions?: RuntimeResolutionOptions): ParseRunConfigResult;