// Hand-written declarations for the plain-JS evaluation engine, so the
// TypeScript connector and CLI get checked types across the package boundary.
// The runtime modules live beside this file.

export type GraderName = "exact" | "json-fields" | "llm-judge" | "script";

export interface EvalCase {
  id: string;
  input: string;
  expected?: unknown;
  grader: GraderName;
  fields?: string[];
  rubric?: string;
  mode?: "document" | "agent";
  workspace?: string;
  graderScript?: string;
  synthetic?: boolean;
}

export interface SkillDescriptor {
  name?: string;
  text: string;
}

export interface AgentUsage {
  input_tokens?: number;
  output_tokens?: number;
  [key: string]: unknown;
}

export interface RunMetrics {
  latencyMs: number;
  estimatedCostUsd: number | null;
  usage: AgentUsage | null;
}

export interface RunResult {
  output: unknown;
  rationale: string;
  metrics: RunMetrics;
}

export interface RunnerArgs {
  agent: "claude" | "codex";
  model?: string;
  timeout: number;
  skill: string;
  input: string;
  mode?: "document" | "agent";
  workspace?: string;
}

export type Runner = (args: RunnerArgs) => Promise<RunResult>;

export interface GradeResult {
  pass: boolean;
  fields: Record<string, boolean>;
  judgeReason?: string;
  judgeMetrics?: RunMetrics;
  detail?: string;
}

export interface EvalRecord {
  id: string;
  skill: string;
  run: number;
  expected: unknown;
  output: unknown;
  rationale: string;
  metrics: RunMetrics | null;
  grade: GradeResult;
  error?: string;
}

export interface SkillSummary {
  skill: string;
  attempts: number;
  caseCount: number;
  errors: number;
  passRate: number | null;
  minPassRate: number;
  runPassRates: number[];
  fields: Record<string, number | null>;
  meanLatencyMs: number | null;
  estimatedCostUsd: number | null;
  costObservations: number;
}

export interface LeaderboardCondition {
  skill: string;
  mean: number | null;
  ci: [number | null, number | null];
  lift: number | null;
  liftCi: [number | null, number | null];
  runs: number;
  tasks: number;
  errors: number;
  tokens: number | null;
  meanLatencyMs: number | null;
}

export interface Leaderboard {
  baseline: string;
  resamples: number;
  conditions: LeaderboardCondition[];
}

export interface EvalReport {
  version: 1;
  createdAt: string;
  agent: string;
  model: string | null;
  runs: number;
  synthetic: boolean;
  records: EvalRecord[];
  summary: SkillSummary[];
  comparison: unknown[];
  leaderboard?: Leaderboard;
}

export interface EvaluateOptions {
  skills: Array<string | SkillDescriptor>;
  cases: EvalCase[];
  runs?: number;
  agent?: "claude" | "codex";
  model?: string;
  timeout?: number;
  concurrency?: number;
  runner?: Runner;
  onProgress?: (done: number, total: number, record: EvalRecord) => void;
}

export function evaluate(options: EvaluateOptions): Promise<EvalReport>;
export function writeReport(report: EvalReport, directory: string): Promise<{ json: string; html: string }>;
export function leaderboard(records: EvalRecord[], options?: { baseline?: string; resamples?: number; seed?: number }): Leaderboard;
export function readCases(filename: string): Promise<EvalCase[]>;
export function validateCases(cases: EvalCase[]): EvalCase[];
export function runAgent(args: RunnerArgs & { workRoot?: string }): Promise<RunResult>;
export function grade(testCase: EvalCase, output: unknown, judge?: (payload: unknown) => Promise<RunResult>): Promise<GradeResult>;
export const BENCHMARK_SUBCOMMANDS: string[];
/** SPEC-v1 benchmark subcommands (screen, heldout, judge, aggregate, check, router). Resolves to the exit code. */
export function runBenchmarkCli(argv: string[]): Promise<number>;