/******************************************************************************
 * Copyright 2024 - 2025 TypeFox GmbH
 * This program and the accompanying materials are made available under the
 * terms of the MIT License, which is available in the project root.
 ******************************************************************************/
import { Evaluator, type EvaluatorResult } from './evaluator.js';
import { type Message } from './message.js';
import { type EvalCase } from './eval-case.js';
/**
 * Configuration for the evaluation matrix
 */
export interface EvalMatrixConfig {
    config: {
        /**
         * Name of the evaluation matrix
         */
        name: string;
        /**
         * Helpful description of the evaluation matrix
         */
        description: string;
        /**
         * Where to store run history
         */
        history_folder: string;
        /**
         * The number of runs to perform for each case
         * Note this will trigger evaluation for all registered evaluators for each run
         */
        num_runs: number;
    };
    /**
     * Runners to evaluate
     */
    runners: Runner[];
    /**
     * Evaluators to evaluate with
     */
    evaluators: NamedEvaluator[];
    /**
     * Cases to evaluate
     */
    cases: EvalCase[];
}
/**
 * Evaluation matrix for running multiple runners on multiple cases with multiple evaluators
 */
export declare class EvalMatrix {
    private config;
    constructor(config: EvalMatrixConfig);
    /**
     * Run the evaluation matrix, getting all results back
     */
    run(): Promise<EvaluatorResult[]>;
}
/**
 * Runner interface for running a prompt against a model, a service, or something else that provides a response
 */
export interface Runner {
    name: string;
    runner: (prompt: string, messages: Message[]) => Promise<string>;
}
/**
 * Generic evaluator interface w/ a name to identify it
 */
export interface NamedEvaluator {
    name: string;
    eval: Evaluator;
}
//# sourceMappingURL=eval-matrix.d.ts.map