import { executeNativeAgentCompare, type NativeAgentCompareResult } from '../compare.js';
import { type BenchmarkEnvironment, type BenchmarkExpectedEnvironment } from './environment.js';
import { type GenerateGraphOptions, type GenerateGraphResult } from '../generate.js';
export type BenchmarkSuiteMode = 'cold' | 'warm' | 'all';
export type BenchmarkSuiteEntryStatus = 'ready' | 'planned';
export interface BenchmarkSuiteRepoPathSource {
    kind: 'path';
    path: string;
}
export interface BenchmarkSuiteRepoGitSource {
    kind: 'git';
    url: string;
    ref?: string;
}
export type BenchmarkSuiteRepoSource = BenchmarkSuiteRepoPathSource | BenchmarkSuiteRepoGitSource;
export interface BenchmarkSuiteRepo {
    id: string;
    name: string;
    path?: string;
    source?: BenchmarkSuiteRepoSource;
    graphRoot?: string;
    description: string;
    size: 'small' | 'mid' | 'large';
    language: string;
    shape: string;
    status: BenchmarkSuiteEntryStatus;
    supportsSpi: boolean;
}
export interface BenchmarkSuiteTask {
    id: string;
    name: string;
    description: string;
    status: BenchmarkSuiteEntryStatus;
    prompts: Record<string, string>;
}
export interface BenchmarkSuiteRunOptions {
    repo: string | null;
    task: string | null;
    reposManifestPath?: string | null;
    tasksManifestPath?: string | null;
    mode: BenchmarkSuiteMode;
    trials: number;
    outputDir: string;
    execTemplate: string;
    dryRun: boolean;
    yes: boolean;
}
interface BenchmarkSuiteMetricStats {
    median: number;
    min: number;
    max: number;
    n: number;
}
interface BenchmarkSuiteArmMetricsSummary {
    input_tokens: BenchmarkSuiteMetricStats | null;
    total_tool_calls: BenchmarkSuiteMetricStats | null;
    read_calls: BenchmarkSuiteMetricStats | null;
    glob_grep_calls: BenchmarkSuiteMetricStats | null;
    wall_clock_ms: BenchmarkSuiteMetricStats | null;
    cost_usd: BenchmarkSuiteMetricStats | null;
}
interface BenchmarkSuitePassFailSummary {
    passed: number;
    failed: number;
    n: number;
}
interface BenchmarkSuiteYesNoSummary {
    yes: number;
    no: number;
    n: number;
}
interface BenchmarkSuiteWorkflowOutcomeSummary {
    wrong_file_edits: BenchmarkSuiteMetricStats | null;
    validation_passed: BenchmarkSuitePassFailSummary | null;
    review_time_seconds: BenchmarkSuiteMetricStats | null;
    rework_loops: BenchmarkSuiteMetricStats | null;
    human_intervention_required: BenchmarkSuiteYesNoSummary | null;
    evidence: string[];
}
interface BenchmarkSuiteWorkflowOutcomeArms {
    legacy: BenchmarkSuiteWorkflowOutcomeSummary | null;
    spi_madar: BenchmarkSuiteWorkflowOutcomeSummary | null;
}
interface BenchmarkSuiteBenchmarkOutcomeCounts {
    full_win: number;
    partial_win: number;
    regression: number;
    not_measured: number;
}
interface BenchmarkSuiteBenchmarkOutcomeSummary {
    counts: BenchmarkSuiteBenchmarkOutcomeCounts;
    evidence: string[];
}
interface BenchmarkSuiteBenchmarkOutcomeArms {
    legacy: BenchmarkSuiteBenchmarkOutcomeSummary | null;
    spi_madar: BenchmarkSuiteBenchmarkOutcomeSummary | null;
}
export interface BenchmarkSuiteSummaryCell {
    repoId: string;
    repoName: string;
    taskId: string;
    taskName: string;
    mode: 'cold' | 'warm';
    prompt: string | null;
    status: 'completed' | 'partial' | 'planned' | 'skipped' | 'env_mismatch';
    reason: string | null;
    isolation: boolean | null;
    baseline: BenchmarkSuiteArmMetricsSummary;
    madar: BenchmarkSuiteArmMetricsSummary;
    spi_madar: BenchmarkSuiteArmMetricsSummary | null;
    benchmark_outcomes: BenchmarkSuiteBenchmarkOutcomeArms | null;
    workflow_outcomes: BenchmarkSuiteWorkflowOutcomeArms | null;
    artifacts: {
        legacy_share_safe_reports: string[];
        spi_share_safe_reports: string[];
    };
}
export interface BenchmarkSuiteSummary {
    schema_version: 1;
    started_at: string;
    completed_at: string;
    output_root: string;
    runtime_artifact: {
        source: string;
        package_version: string | null;
        tarball_name: string | null;
        tarball_sha256: string | null;
    };
    filters: {
        repo: string | null;
        task: string | null;
        repos_manifest: string | null;
        tasks_manifest: string | null;
        mode: BenchmarkSuiteMode;
        trials: number;
    };
    cells_skipped_for_install: number;
    cells_skipped_for_env_drift: number;
    cells: BenchmarkSuiteSummaryCell[];
}
export interface BenchmarkSuiteRunResult {
    text: string;
    outputRoot?: string;
    summaryPath?: string;
    summaryJsonPath?: string;
    summary?: BenchmarkSuiteSummary;
}
export interface BenchmarkSuiteDependencies {
    repos?: BenchmarkSuiteRepo[];
    tasks?: BenchmarkSuiteTask[];
    tasksPath?: string;
    now?: () => Date;
    generateGraph?: (rootPath?: string, options?: GenerateGraphOptions) => GenerateGraphResult;
    captureBenchmarkEnvironment?: (options: {
        projectRoot: string;
    }) => Promise<BenchmarkEnvironment>;
    executeNativeAgentCompare?: (input: Parameters<typeof executeNativeAgentCompare>[0]) => Promise<NativeAgentCompareResult>;
    expectedEnvironment?: BenchmarkExpectedEnvironment | null;
}
export declare function loadBenchmarkSuiteRepos(path?: string): BenchmarkSuiteRepo[];
export declare function loadBenchmarkSuiteTasks(path?: string): BenchmarkSuiteTask[];
export declare function runBenchmarkSuite(options: BenchmarkSuiteRunOptions, dependencies?: BenchmarkSuiteDependencies): Promise<BenchmarkSuiteRunResult>;
export {};