export interface BenchmarkSample {
  taskId: string;
  cohort: string;
  system: "baseline" | "ultrapi";
  verified: boolean;
  costToGreen: number;
  timeToGreenMs: number;
}

export interface CohortBenchmark {
  cohort: string;
  pairs: number;
  baselineVerifiedSuccessRate: number;
  ultraVerifiedSuccessRate: number;
  costChange: number;
  timeChange: number;
  qualifies: boolean;
}

export interface BenchmarkReport {
  passed: boolean;
  tolerance: number;
  verifiedSuccessRate: { baseline: number; ultrapi: number };
  cohorts: CohortBenchmark[];
}

export interface BenchmarkDataset { samples: BenchmarkSample[]; synthetic: boolean; note?: string }

/**
 * Accepts either a bare array of collected samples or a document that declares itself. A
 * dataset that carries `synthetic: true` is a fixture for this evaluator and says so out loud,
 * so no reader and no script can mistake invented rows for a measurement.
 */
export function parseBenchmarkDataset(value: unknown): BenchmarkDataset {
  if (Array.isArray(value)) return { samples: value as BenchmarkSample[], synthetic: false };
  if (!value || typeof value !== "object") throw new Error("Benchmark dataset must be an array of samples or an object with a samples array");
  const document = value as { samples?: unknown; synthetic?: unknown; note?: unknown };
  if (!Array.isArray(document.samples)) throw new Error("Benchmark dataset object must have a samples array");
  return { samples: document.samples as BenchmarkSample[], synthetic: document.synthetic === true, ...(typeof document.note === "string" ? { note: document.note } : {}) };
}

function mean(values: number[]): number { return values.reduce((sum, value) => sum + value, 0) / values.length; }
function rate(samples: BenchmarkSample[]): number { return samples.filter((sample) => sample.verified).length / samples.length; }
function relativeChange(baseline: number, ultrapi: number): number { return baseline === 0 ? ultrapi === 0 ? 0 : Infinity : ultrapi / baseline - 1; }

export function evaluateBenchmark(samples: readonly BenchmarkSample[], tolerance = 0.05): BenchmarkReport {
  if (!Number.isFinite(tolerance) || tolerance < 0) throw new Error("Benchmark tolerance must be non-negative");
  if (samples.length === 0) throw new Error("Benchmark dataset is empty");
  const keys = new Set<string>();
  for (const sample of samples) {
    if (!sample.taskId || !sample.cohort || !["baseline", "ultrapi"].includes(sample.system)) throw new Error("Benchmark sample is malformed");
    if (![sample.costToGreen, sample.timeToGreenMs].every((value) => Number.isFinite(value) && value >= 0)) throw new Error("Benchmark metrics must be non-negative");
    const key = `${sample.taskId}\u0000${sample.system}`;
    if (keys.has(key)) throw new Error(`Duplicate benchmark sample ${sample.taskId}/${sample.system}`);
    keys.add(key);
  }
  const baseline = samples.filter((sample) => sample.system === "baseline");
  const ultrapi = samples.filter((sample) => sample.system === "ultrapi");
  if (baseline.length === 0 || ultrapi.length === 0) throw new Error("Benchmark requires baseline and UltraPi samples");
  const cohorts = [...new Set(samples.map((sample) => sample.cohort))].sort().map((cohort): CohortBenchmark => {
    const base = baseline.filter((sample) => sample.cohort === cohort);
    const ultra = ultrapi.filter((sample) => sample.cohort === cohort);
    const taskIds = base.map((sample) => sample.taskId).filter((taskId) => ultra.some((sample) => sample.taskId === taskId));
    if (taskIds.length < 5 || taskIds.length !== base.length || taskIds.length !== ultra.length) throw new Error(`Cohort ${cohort} requires at least five complete comparable pairs`);
    const baseVerified = base.filter((sample) => taskIds.includes(sample.taskId));
    const ultraVerified = ultra.filter((sample) => taskIds.includes(sample.taskId));
    const costChange = relativeChange(mean(baseVerified.map((sample) => sample.costToGreen)), mean(ultraVerified.map((sample) => sample.costToGreen)));
    const timeChange = relativeChange(mean(baseVerified.map((sample) => sample.timeToGreenMs)), mean(ultraVerified.map((sample) => sample.timeToGreenMs)));
    const baselineVerifiedSuccessRate = rate(baseVerified);
    const ultraVerifiedSuccessRate = rate(ultraVerified);
    return {
      cohort,
      pairs: taskIds.length,
      baselineVerifiedSuccessRate,
      ultraVerifiedSuccessRate,
      costChange,
      timeChange,
      qualifies: ultraVerifiedSuccessRate >= baselineVerifiedSuccessRate && ((costChange < 0 && timeChange <= tolerance) || (timeChange < 0 && costChange <= tolerance)),
    };
  });
  const verifiedSuccessRate = { baseline: rate(baseline), ultrapi: rate(ultrapi) };
  return { passed: verifiedSuccessRate.ultrapi >= verifiedSuccessRate.baseline && cohorts.some((cohort) => cohort.qualifies), tolerance, verifiedSuccessRate, cohorts };
}