import type { PercentileMetric, RateMetric } from "./MetricTypes.js";
import type { CodaliGatewayModelTier } from "../gateway/CodaliGatewayTypes.js";
import { CODALI_GATEWAY_DATASET_EVAL_SCHEMA_VERSION, type CodaliGatewayDatasetEvalImportLineage, type CodaliGatewayDatasetEvalStage } from "./GatewayDatasetEval.js";
export type CodaliGatewayEvalTaskType = "generic_question" | "code_repo_question" | "encrypted_docdex_search_question" | "product_tool_question" | "disabled_integration_question" | "image_generation_question" | "missing_evidence_question";
export type CodaliGatewayEvalRunStatus = "passed" | "failed";
export interface CodaliGatewayEvalDatasetSchemaVersions {
    gatewayEval: 1;
    datasetEval: typeof CODALI_GATEWAY_DATASET_EVAL_SCHEMA_VERSION;
    storageContract: string;
    datasetReplayFixture: string;
    datasetRecord: string;
}
export interface CodaliGatewayEvalDatasetMetadata {
    source: "dataset_replay_fixture";
    stage: CodaliGatewayDatasetEvalStage;
    sourceRecordId: string;
    sourceGatewayRecordId?: string;
    datasetKind: string;
    promptVersion: string;
    schemaVersions: CodaliGatewayEvalDatasetSchemaVersions;
    sourceObjectHashes: string[];
    replayFixtureId?: string;
    exportKind?: string;
    generatedAt?: string;
}
export interface CodaliGatewayEvalCaseExpectations {
    allowedTools?: string[];
    deniedTools?: string[];
    requiredTools?: string[];
    requiredSourceTypes?: string[];
    requiresEvidence?: boolean;
    expectsMissingEvidence?: boolean;
    requiresImageArtifact?: boolean;
    requiresFinalLargeModel?: boolean;
    minEvidencePrecision?: number;
    maxLatencyMs?: number;
    maxTokens?: number;
    maxToolCalls?: number;
    maxModelCalls?: number;
    maxCostUsd?: number;
}
export interface CodaliGatewayEvalCase {
    id: string;
    type: CodaliGatewayEvalTaskType;
    prompt: string;
    expectations: CodaliGatewayEvalCaseExpectations;
    dataset?: CodaliGatewayEvalDatasetMetadata;
}
export interface CodaliGatewayEvalEvidenceRecord {
    id: string;
    sourceType: string;
    sourceId?: string;
    cited?: boolean;
    relevant?: boolean;
}
export interface CodaliGatewayEvalCitationRecord {
    evidenceId?: string;
    sourceType?: string;
    sourceId?: string;
}
export interface CodaliGatewayEvalImageArtifactRecord {
    id?: string;
    uri?: string;
    mimeType?: string;
    metadata?: Record<string, unknown>;
}
export interface CodaliGatewayEvalRunRecord {
    caseId: string;
    taskType: CodaliGatewayEvalTaskType;
    status: CodaliGatewayEvalRunStatus;
    plannerSchemaValid: boolean;
    selectedTaskType?: CodaliGatewayEvalTaskType;
    calledTools: string[];
    evidence: CodaliGatewayEvalEvidenceRecord[];
    citations: CodaliGatewayEvalCitationRecord[];
    finalAnswer: string;
    finalAnswerDirect?: boolean;
    finalModelTier?: CodaliGatewayModelTier;
    latencyMs?: number;
    tokensUsed?: number;
    costUsd?: number;
    toolCallCount?: number;
    modelCallCount?: number;
    imageArtifact?: CodaliGatewayEvalImageArtifactRecord;
    missingEvidenceHandled?: boolean;
    warnings?: string[];
    errors?: string[];
    metadata?: Record<string, unknown>;
}
export interface CodaliGatewayEvalCaseResult {
    caseId: string;
    taskType: CodaliGatewayEvalTaskType;
    status: CodaliGatewayEvalRunStatus;
    passed: boolean;
    plannerSchemaValid: boolean;
    evidencePrecision: number | null;
    evidencePrecisionPassed: boolean;
    citationSourceCorrect: boolean;
    disabledToolLeakageDetected: boolean;
    finalAnswerDirect: boolean;
    finalLargeModelUsed: boolean;
    budgetCompliant: boolean;
    imageArtifactPresent: boolean;
    missingEvidenceHandled: boolean;
    calledTools: string[];
    failures: string[];
    warnings: string[];
    errors: string[];
    latencyMs?: number;
    tokensUsed?: number;
    costUsd?: number;
    toolCallCount?: number;
    modelCallCount?: number;
    dataset?: CodaliGatewayEvalDatasetMetadata;
}
export interface CodaliGatewayEvalMetrics {
    schemaVersion: 1;
    generatedAt: string;
    taskCount: number;
    plannerSchemaValidityRate: RateMetric;
    evidencePrecisionRate: RateMetric;
    citationSourceCorrectnessRate: RateMetric;
    disabledToolLeakageRate: RateMetric;
    finalAnswerDirectnessRate: RateMetric;
    finalLargeModelRate: RateMetric;
    budgetComplianceRate: RateMetric;
    latencyMs: PercentileMetric;
    tokensUsed: PercentileMetric;
    costUsd: PercentileMetric;
    toolCallCount: PercentileMetric;
    modelCallCount: PercentileMetric;
    datasetCaseCount: number;
    datasetStageCoverageRate: RateMetric;
    datasetLineageCoverageRate: RateMetric;
    promptSchemaVersionCoverageRate: RateMetric;
}
export interface CodaliGatewayEvalThresholds {
    plannerSchemaValidityMin: number;
    evidencePrecisionMin: number;
    citationSourceCorrectnessMin: number;
    disabledToolLeakageMax: number;
    finalAnswerDirectnessMin: number;
    finalLargeModelRateMin: number;
    budgetComplianceMin: number;
    latencyP95MsMax?: number;
    costP95UsdMax?: number;
    latencyRegressionRatioMax: number;
    costRegressionRatioMax: number;
    datasetStageCoverageMin: number;
    datasetLineageCoverageMin: number;
    promptSchemaVersionCoverageMin: number;
}
export interface CodaliGatewayEvalMetricDelta {
    key: "latency_ms.p95" | "cost_usd.p95" | "tokens_used.p95";
    unit: "ms" | "usd" | "tokens";
    baseline: number | null;
    current: number | null;
    delta: number | null;
    relativeDelta: number | null;
    regression: boolean;
}
export interface CodaliGatewayEvalRegressionComparison {
    schemaVersion: 1;
    status: "baseline_missing" | "compared";
    baselineReportId?: string;
    baselineGeneratedAt?: string;
    deltas: CodaliGatewayEvalMetricDelta[];
    regressionCount: number;
}
export interface CodaliGatewayEvalGateFailure {
    code: string;
    metric: string;
    message: string;
    threshold: number;
    actual: number | null;
    baseline?: number | null;
    delta?: number | null;
}
export interface CodaliGatewayEvalGateResult {
    schemaVersion: 1;
    passed: boolean;
    thresholds: CodaliGatewayEvalThresholds;
    failures: CodaliGatewayEvalGateFailure[];
}
export interface CodaliGatewayEvalReportLineage {
    schemaVersion: 1;
    source: "static_cases" | "dataset_replay_fixture" | "mixed";
    staticCaseIds: string[];
    dataset?: CodaliGatewayDatasetEvalImportLineage;
    datasetSourceRecordIds: string[];
    datasetSourceGatewayRecordIds: string[];
    datasetSourceObjectHashes: string[];
    datasetStageCounts: Record<CodaliGatewayDatasetEvalStage, number>;
}
export interface CodaliGatewayEvalReportVersions {
    schemaVersion: 1;
    gatewayEval: 1;
    datasetEval: typeof CODALI_GATEWAY_DATASET_EVAL_SCHEMA_VERSION;
    promptVersions: Record<string, string>;
    schemaVersions: Record<string, string | number>;
}
export interface CodaliGatewayEvalReport {
    schemaVersion: 1;
    reportId: string;
    runId: string;
    runtime: "codali_gateway_eval";
    mode: "gateway_smoke";
    startedAt: string;
    endedAt: string;
    durationMs: number;
    cases: CodaliGatewayEvalCaseResult[];
    metrics: CodaliGatewayEvalMetrics;
    regression: CodaliGatewayEvalRegressionComparison;
    gates: CodaliGatewayEvalGateResult;
    lineage: CodaliGatewayEvalReportLineage;
    versions: CodaliGatewayEvalReportVersions;
    summary: {
        status: CodaliGatewayEvalRunStatus;
        total: number;
        passed: number;
        failed: number;
    };
    warnings: string[];
    errors: string[];
}
export type CodaliGatewayEvalRunner = (evalCase: CodaliGatewayEvalCase) => Promise<CodaliGatewayEvalRunRecord>;
export interface CodaliGatewayEvalSuiteOptions {
    runId?: string;
    reportId?: string;
    cases?: CodaliGatewayEvalCase[];
    runner?: CodaliGatewayEvalRunner;
    thresholds?: Partial<CodaliGatewayEvalThresholds>;
    baseline?: CodaliGatewayEvalReport | CodaliGatewayEvalMetrics;
    datasetCases?: CodaliGatewayEvalCase[];
    datasetLineage?: CodaliGatewayDatasetEvalImportLineage;
    includeDatasetCases?: boolean;
}
export declare const DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS: CodaliGatewayEvalThresholds;
export declare const CODALI_GATEWAY_EVAL_CASES: CodaliGatewayEvalCase[];
export declare const evaluateCodaliGatewayEvalCase: (evalCase: CodaliGatewayEvalCase, record: CodaliGatewayEvalRunRecord, thresholds?: CodaliGatewayEvalThresholds) => CodaliGatewayEvalCaseResult;
export declare const aggregateCodaliGatewayEvalMetrics: (cases: CodaliGatewayEvalCaseResult[]) => CodaliGatewayEvalMetrics;
export declare const compareCodaliGatewayEvalBaseline: (params: {
    current: CodaliGatewayEvalMetrics;
    baseline?: CodaliGatewayEvalReport | CodaliGatewayEvalMetrics;
}) => CodaliGatewayEvalRegressionComparison;
export declare const evaluateCodaliGatewayEvalGates: (params: {
    metrics: CodaliGatewayEvalMetrics;
    thresholds?: Partial<CodaliGatewayEvalThresholds>;
    regression?: CodaliGatewayEvalRegressionComparison;
}) => CodaliGatewayEvalGateResult;
export declare const createDefaultCodaliGatewayEvalRunner: () => CodaliGatewayEvalRunner;
export declare const runCodaliGatewayEvalSuite: (options?: CodaliGatewayEvalSuiteOptions) => Promise<CodaliGatewayEvalReport>;
export declare const formatCodaliGatewayEvalTextReport: (report: CodaliGatewayEvalReport) => string;
//# sourceMappingURL=GatewayEvalSuite.d.ts.map