import type { MetricFn, MetricConfig } from './metrics.js';
export interface EvalCase {
    id: string;
    description: string;
    input: string;
    expected: unknown;
    metrics: Array<{
        name: string;
        fn: MetricFn;
        config?: MetricConfig;
        weight?: number;
    }>;
    expectedToolCalls?: string[];
    outputSchema?: Record<string, unknown>;
    tags?: string[];
}
export interface EvalSuiteConfig {
    name: string;
    description?: string;
    cases: EvalCase[];
    defaultMetrics?: Array<{
        name: string;
        fn: MetricFn;
        config?: MetricConfig;
        weight?: number;
    }>;
}
export interface EvalMetric {
    name: string;
    score: number;
    weight: number;
}
export interface EvalResult {
    caseId: string;
    description: string;
    response: string;
    metrics: EvalMetric[];
    overallScore: number;
    passed: boolean;
    toolCallsUsed?: string[];
    durationMs?: number;
}
export interface EvalReport {
    suiteName: string;
    description?: string;
    timestamp: string;
    results: EvalResult[];
    summary: {
        totalCases: number;
        passed: number;
        failed: number;
        averageScore: number;
        minScore: number;
        maxScore: number;
        metrics: Record<string, {
            avg: number;
            min: number;
            max: number;
        }>;
    };
}
export interface EvalRunnerOptions {
    passThreshold?: number;
    includeResponses?: boolean;
}
export declare class EvalSuite {
    readonly name: string;
    readonly description: string;
    readonly cases: EvalCase[];
    readonly defaultMetrics: EvalSuiteConfig['defaultMetrics'];
    constructor(config: EvalSuiteConfig);
    getMetricsForCase(testCase: EvalCase): Array<{
        name: string;
        fn: MetricFn;
        config?: MetricConfig;
        weight: number;
    }>;
}
export declare class EvalRunner {
    private passThreshold;
    private includeResponses;
    constructor(options?: EvalRunnerOptions);
    runSuite(suite: EvalSuite, generateResponse: (input: string, testCase: EvalCase) => Promise<{
        response: string;
        toolCalls?: string[];
        durationMs?: number;
    }>): Promise<EvalReport>;
    evaluateCase(testCase: EvalCase, response: string, toolCallsUsed?: string[], durationMs?: number, suite?: EvalSuite): EvalResult;
    private buildReport;
    static formatReport(report: EvalReport): string;
    static formatReportJson(report: EvalReport): string;
}
//# sourceMappingURL=eval-runner.d.ts.map