/**
 * Model Benchmarking Types
 * Type definitions for performance testing and evaluation
 */
/**
 * Benchmark test status
 */
export declare enum BenchmarkStatus {
    PENDING = "pending",
    RUNNING = "running",
    COMPLETED = "completed",
    FAILED = "failed",
    CANCELLED = "cancelled"
}
/**
 * Metric comparison operator
 */
export declare enum ComparisonOperator {
    GREATER_THAN = "gt",
    LESS_THAN = "lt",
    EQUAL = "eq",
    GREATER_THAN_OR_EQUAL = "gte",
    LESS_THAN_OR_EQUAL = "lte"
}
/**
 * Benchmark test result status
 */
export declare enum TestResultStatus {
    PASSED = "passed",
    FAILED = "failed",
    SKIPPED = "skipped",
    ERROR = "error"
}
/**
 * Benchmark metric type
 */
export declare enum MetricType {
    LATENCY = "latency",
    THROUGHPUT = "throughput",
    ACCURACY = "accuracy",
    COST = "cost",
    TOKENS = "tokens",
    ERROR_RATE = "error_rate",
    CUSTOM = "custom"
}
/**
 * Test case for benchmarking
 */
export interface BenchmarkTestCase {
    id: string;
    name: string;
    description?: string;
    input: {
        prompt?: string;
        messages?: Array<{
            role: string;
            content: string;
        }>;
        params?: Record<string, unknown>;
    };
    expectedOutput?: {
        contains?: string[];
        notContains?: string[];
        pattern?: RegExp;
        exactMatch?: string;
        validator?: (output: string) => boolean;
    };
    tags?: string[];
    weight?: number;
    timeout?: number;
    metadata?: Record<string, unknown>;
}
/**
 * Dataset for benchmark tests
 */
export interface BenchmarkDataset {
    id: string;
    name: string;
    description?: string;
    testCases: BenchmarkTestCase[];
    metadata?: Record<string, unknown>;
}
/**
 * Model configuration for benchmarking
 */
export interface BenchmarkModelConfig {
    modelId: string;
    provider: string;
    params?: Record<string, unknown>;
    apiKey?: string;
    endpoint?: string;
}
/**
 * Performance assertion for test validation
 */
export interface PerformanceAssertion {
    metric: MetricType | string;
    operator: ComparisonOperator;
    value: number;
    message?: string;
}
/**
 * Benchmark test configuration
 */
export interface BenchmarkConfig {
    models: BenchmarkModelConfig[];
    dataset: BenchmarkDataset | string;
    runs?: number;
    parallel?: boolean;
    maxConcurrency?: number;
    assertions?: PerformanceAssertion[];
    timeout?: number;
    warmup?: boolean;
    compareBaseline?: boolean;
    baselineModelId?: string;
    saveResults?: boolean;
    outputFormat?: 'json' | 'csv' | 'html' | 'markdown';
    outputPath?: string;
    metadata?: Record<string, unknown>;
}
/**
 * Result for a single test case execution
 */
export interface TestCaseResult {
    testCaseId: string;
    modelId: string;
    status: TestResultStatus;
    input: BenchmarkTestCase['input'];
    output?: string;
    expectedOutput?: BenchmarkTestCase['expectedOutput'];
    metrics: {
        latency: number;
        tokens?: {
            prompt: number;
            completion: number;
            total: number;
        };
        cost?: number;
        custom?: Record<string, number>;
    };
    error?: {
        message: string;
        code?: string;
        stack?: string;
    };
    timestamp: Date;
    duration: number;
    run: number;
}
/**
 * Aggregated metrics for a model
 */
export interface ModelMetrics {
    modelId: string;
    testCases: number;
    passed: number;
    failed: number;
    errors: number;
    successRate: number;
    latency: {
        mean: number;
        median: number;
        p50: number;
        p95: number;
        p99: number;
        min: number;
        max: number;
        stdDev: number;
    };
    throughput: number;
    tokens: {
        totalPrompt: number;
        totalCompletion: number;
        total: number;
        avgPerRequest: number;
    };
    cost: {
        total: number;
        avgPerRequest: number;
        currency: string;
    };
    accuracy?: {
        score: number;
        correctPredictions: number;
        totalPredictions: number;
    };
    errorRate: number;
    customMetrics?: Record<string, number>;
}
/**
 * Comparison between two models
 */
export interface ModelComparison {
    modelA: string;
    modelB: string;
    winner?: string;
    metrics: {
        [key: string]: {
            modelA: number;
            modelB: number;
            difference: number;
            percentDifference: number;
            better: string;
        };
    };
}
/**
 * Benchmark run result
 */
export interface BenchmarkResult {
    id: string;
    config: BenchmarkConfig;
    status: BenchmarkStatus;
    startedAt: Date;
    completedAt?: Date;
    duration?: number;
    testCaseResults: TestCaseResult[];
    modelMetrics: Record<string, ModelMetrics>;
    comparisons?: ModelComparison[];
    assertionResults?: Array<{
        assertion: PerformanceAssertion;
        passed: boolean;
        actualValue: number;
        message: string;
    }>;
    summary: {
        totalTests: number;
        totalModels: number;
        totalRuns: number;
        passRate: number;
        bestModel: {
            byLatency: string;
            byAccuracy?: string;
            byCost: string;
            byThroughput: string;
        };
    };
    errors?: Array<{
        message: string;
        testCaseId?: string;
        modelId?: string;
        timestamp: Date;
    }>;
    metadata?: Record<string, unknown>;
}
/**
 * Benchmark progress information
 */
export interface BenchmarkProgress {
    benchmarkId: string;
    status: BenchmarkStatus;
    totalTests: number;
    completedTests: number;
    failedTests: number;
    progress: number;
    currentModel?: string;
    currentTestCase?: string;
    elapsedTime: number;
    estimatedTimeRemaining?: number;
}
/**
 * Historical benchmark result summary
 */
export interface BenchmarkHistory {
    benchmarkId: string;
    timestamp: Date;
    models: string[];
    dataset: string;
    summary: BenchmarkResult['summary'];
    duration: number;
}
/**
 * Baseline performance data
 */
export interface BaselineData {
    modelId: string;
    dataset: string;
    metrics: ModelMetrics;
    timestamp: Date;
    version?: string;
}
//# sourceMappingURL=types.d.ts.map