import type { Artifact, BatchEvaluationReport, EvaluationReport, JobStore, ProgressCallback, Report, VarianceData, VariantSpec } from '../types/index.js';
export interface SkillProgressInfo {
    phase: string;
    skill: string;
    current: number;
    total: number;
}
interface CommonEvaluationOptions {
    /** Core API is runtime-neutral: callers must select the measured model explicitly. */
    model: string;
    outputDir?: string | null;
    project?: string;
    owner?: string;
    tags?: string[];
    noJudge?: boolean;
    concurrency?: number;
    timeoutMs?: number;
    /** Core API is runtime-neutral: CLI/default resolution happens before this boundary. */
    executorName: string;
    jobStore?: JobStore | null;
    persistJob?: boolean;
    onProgress?: ProgressCallback | null;
    /** 跳过 LLM 模型连通性检测。仅当 --resume 报告通过完整契约校验时自动 true。 */
    skipConnectivity?: boolean;
    /** 跳过 doctor 健康检查门禁(escape hatch, 默认 false)。
     *  开启后 doctor 整段不跑,失败也不阻断 eval — 评测环境用 mock/stub 提供
     *  依赖时绕开 doctor 的物理路径检查误报。开启意味着接受 garbage-in 风险。 */
    skipDoctor?: boolean;
    /** 用户语言, 透传给 doctor 报告渲染。 */
    lang?: 'zh' | 'en';
    mcpConfig?: string;
    verbose?: boolean;
    layeredStats?: boolean;
    /** --repeat N. 1 表示单次(默认); > 1 时在 runMultiple 层聚合 variance。
     *  记入 report.meta.request.repeat 让 meta 如实反映用户输入。 */
    repeat?: number;
    /** --holdout-ratio R. 0 / 缺省 = 不切分(默认)。> 0 时 report-finalize 算 train/holdout
     *  子集综合分(report.analysis.holdout),供 verdict 过拟合门控读取。 */
    holdoutRatio?: number;
    /** --batch 模式标记, true 表示当前评测是 skill batch 流程。
     *  记入 report.meta.request.batch。 */
    batch?: boolean;
    /** --judge-repeat N. 每条 sample × dimension 调 LLM judge N 次, 输出 stddev (judge 自一致性).
     *  默认 1 (单次). 用于量化 LLM judge 在该 rubric 上的稳定性 — stddev 高 = 评分噪声大. */
    judgeRepeat?: number;
    /** Unified judge config. 1 entry = single judge, ≥ 2 = ensemble + inter-judge agreement.
     *  Programmatic callers that omit it reuse the measured executor/model. The CLI
     *  resolves provider-specific economical defaults before crossing this boundary. */
    judgeModels?: import('../types/index.js').JudgeConfig[];
    /** --bootstrap. Distribution-free CI on each variant mean + pairwise diff. */
    bootstrap?: boolean;
    /** --bootstrap-samples N. Default 1000. */
    bootstrapSamples?: number;
    /** length-debias toggle. Default true (length-debias instruction on).
     *  CLI passes false when --no-debias-length is set. */
    lengthDebias?: boolean;
    /** hard budget caps. */
    budget?: import('../types/index.js').EvalBudget;
    noCache?: boolean;
    /** Reasoning effort for executor LLM。透传到 ExecutorInput.effort。
     *  undefined means the selected executor's own default;
     *  CLI parseRunConfig currently resolves an explicit provider-aware value. */
    effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
    /** 关闭 diagnostic LLM call。Default false。跟 noJudge 完全独立。 */
    noDiagnostic?: boolean;
}
export interface RunEvaluationOptions extends CommonEvaluationOptions {
    samplesPath: string;
    skillDir: string;
    variantSpecs?: VariantSpec[];
    dryRun?: boolean;
    retry?: number;
    resume?: string;
    /** Explicit persisted run id. Used by batch workflows that need stable child ids. */
    runId?: string;
    /** strict-baseline default (CLI `--strict-baseline` default true).
     *  When undefined, treated as true. baseline-kind variants get allowedSkills=[].
     *  per-variant 显式隔离声明走 variantSpecs[].allowedSkills(eval.yaml 经 configVariantsToSpecs)。 */
    strictBaseline?: boolean;
}
export interface RunBatchEvaluationOptions extends CommonEvaluationOptions {
    skillDir: string;
    dryRun?: boolean;
    onSkillProgress?: ((info: SkillProgressInfo) => void) | null;
    /** strict-baseline default. batch mode 仍尊重此 flag。 */
    strictBaseline?: boolean;
    /** per-variant allowedSkills override (rare in batch mode but supported). */
    variantAllowedSkills?: Record<string, string[]>;
}
export interface RunMultipleOptions extends RunEvaluationOptions {
    repeat?: number;
    onRepeatProgress?: ((info: {
        run: number;
        total: number;
    }) => void) | null;
}
interface DryRunTask {
    sample_id: string;
    variant: string;
    artifactKind: Artifact['kind'];
    artifactSource: Artifact['source'];
    executionStrategy: string;
    experimentType: string;
    experimentRole: string;
    cwd: string | null;
    promptPreview: string;
    hasRubric: boolean;
    hasAssertions: boolean;
    hasDimensions: boolean;
    hasSystem: boolean;
}
interface DryRunBase {
    dryRun: true;
    model: string;
    judgeModels: import('../types/index.js').JudgeConfig[];
    executor: string;
    skillDir: string;
    totalTasks: number;
}
export interface DryRunReport extends DryRunBase {
    variants: string[];
    samplesPath: string;
    tasks: DryRunTask[];
}
export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
    report: Report | DryRunReport;
    filePath: string | null;
}>;
interface DryRunBatchSkill {
    name: string;
    samplesPath: string;
    sampleCount: number;
    taskCount: number;
}
export interface DryRunBatchReport extends DryRunBase {
    batch: true;
    totalArtifacts: number;
    artifacts: DryRunBatchSkill[];
}
export declare function buildVarianceData(runs: Report[], bootstrapSamples?: number, seed?: number): VarianceData | null;
export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
    report: BatchEvaluationReport | DryRunBatchReport;
    filePath: string | null;
}>;
export declare function runMultiple({ repeat, onRepeatProgress, ...config }: RunMultipleOptions): Promise<{
    report: EvaluationReport;
    aggregated: VarianceData | null;
    filePath: string | null;
}>;
export {};