/******************************************************************************
 * Copyright 2024 - 2025 TypeFox GmbH
 * This program and the accompanying materials are made available under the
 * terms of the MIT License, which is available in the project root.
 ******************************************************************************/
/**
 * Result from running an evaluator
 */
export type EvaluatorResultData = Record<string, unknown>;
export type EvaluatorResultMetadata = Record<string, unknown> & {
    /**
     * Time it took to run the evaluator case (duration)
     */
    duration: number;
};
/**
 * Evaluator result type
 */
export type EvaluatorResult<T = EvaluatorResultData> = {
    /**
     * Name of this evaluation
     */
    name: string;
    /**
     * Metadata for this evaluation, which at the minimum includes a duration for how long it took to run
     */
    metadata: EvaluatorResultMetadata;
    /**
     * Data for this evaluation
     */
    data: T;
};
/**
 * Helper to process a set of results, averaging all runs of each runner-evaluator-case combination
 * The averaged result will contain numeric results only that can be aggregated
 */
export declare function averageAcrossCases(results: EvaluatorResult[]): EvaluatorResult[];
/**
 * Averages all results across runners at the highest level, to get a single result for each runner
 */
export declare function averageAcrossRunners(results: EvaluatorResult[]): EvaluatorResult[];
/**
 * Report
 */
export interface Report {
    config: {
        name: string;
        description: string;
        history_folder: string;
        num_runs: number;
    };
    date: string;
    runTime: string;
    results: EvaluatorResult[];
}
/**
 * Loads a specific report, containing evaluator results from a file & returns it
 */
export declare function loadReport(file: string): Report;
/**
 * Attempts to load the most recent evaluator results from the given file
 */
export declare function loadLastResults(dir: string, take?: number): EvaluatorResult[];
/**
 * Evaluator class for evaluating agent responses
 */
export declare abstract class Evaluator {
    /**
     * Run an evalution over some response and compare with an expected one.
     * The result is a data entry that will be part of the overall evaluator result.
     */
    abstract evaluate(response: string, expected_response: string): Promise<EvaluatorResultData>;
}
export declare function mergeEvaluators(...evaluators: Evaluator[]): Evaluator;
//# sourceMappingURL=evaluator.d.ts.map