/**
 * Agent A/B Testing Framework - Run A/B tests between agent variants.
 *
 * Supports multiple variants, configurable sample sizes, chi-squared
 * statistical significance testing, and winner recommendation.
 */
import type { SuiteResult } from './types';
export interface AgentVariant {
    name: string;
    model?: string;
    config?: Record<string, any>;
    env?: Record<string, string>;
}
export interface ABTestConfig {
    /** Legacy: two-model comparison */
    modelA?: string;
    modelB?: string;
    /** New: multi-variant support */
    variants?: AgentVariant[];
    suitePath: string;
    runs: number;
    /** Alias for runs */
    sampleSize?: number;
    /** Primary metric to compare: 'passRate' | 'cost' | 'time' */
    metric?: string;
}
export interface ABModelResult {
    model: string;
    variant?: AgentVariant;
    passRate: number;
    avgCost: number;
    avgTime: number;
    passCount: number;
    failCount: number;
    results: SuiteResult[];
}
export interface ABTestResult {
    modelA: ABModelResult;
    modelB: ABModelResult;
    variants: ABModelResult[];
    pValue: number;
    chiSquared: number;
    significant: boolean;
    qualityWinner: string;
    costWinner: string;
    recommendation: string;
}
/**
 * Chi-squared test for independence between variants.
 * Compares observed pass/fail counts against expected (pooled) rates.
 */
export declare function chiSquaredTest(variants: Array<{
    pass: number;
    fail: number;
}>): {
    chiSquared: number;
    pValue: number;
    df: number;
};
export declare function tTest(a: number[], b: number[]): number;
export declare class ABTestRunner {
    private variants;
    private sampleSize;
    readonly metric: string;
    constructor(config: {
        variants: AgentVariant[];
        sampleSize: number;
        metric: string;
    });
    /**
     * Run the A/B test across all variants on the given test suite.
     */
    run(_testSuite: string): ABTestResult;
    /**
     * Run the A/B test asynchronously (actually executes test suites).
     */
    runAsync(testSuitePath: string): Promise<ABTestResult>;
    private buildResult;
    /**
     * Check if results are statistically significant at the given confidence level.
     */
    isSignificant(results: ABTestResult, confidence?: number): boolean;
}
export declare function runABTest(config: ABTestConfig): Promise<ABTestResult>;
/**
 * Format A/B test results for console output.
 */
export declare function formatABTest(result: ABTestResult): string;
//# sourceMappingURL=ab-test.d.ts.map