//#region src/eval/types.d.ts
/**
 * Public types for the minimal inline eval runner. Full orchestrator,
 * scorer libraries, dataset loaders, and reporters live in the
 * separate `@graphorin/evals` package (post-MVP).
 *
 * @packageDocumentation
 */
/**
 * One sample from an eval dataset.
 *
 * @stable
 */
interface Case<I, O = unknown, M = Readonly<Record<string, unknown>>> {
  readonly id?: string;
  readonly input: I;
  readonly expected?: O;
  readonly metadata?: M;
}
/**
 * @stable
 */
interface Dataset<I, O = unknown, M = Readonly<Record<string, unknown>>> {
  readonly cases: ReadonlyArray<Case<I, O, M>>;
  readonly metadata?: {
    readonly name?: string;
    readonly description?: string;
    readonly createdAt?: Date;
  };
}
/**
 * Output of {@link Scorer.score}.
 *
 * @stable
 */
interface ScoreResult {
  readonly pass: boolean;
  /** Optional normalized score in `[0, 1]`. */
  readonly score?: number;
  readonly reason?: string;
  readonly metadata?: Readonly<Record<string, unknown>>;
}
/**
 * @stable
 */
interface Scorer<I, O = unknown> {
  readonly name: string;
  score(args: {
    readonly case: Case<I, O>;
    readonly output: O;
    readonly durationMs: number;
  }): Promise<ScoreResult>;
}
/**
 * Per-case result.
 *
 * @stable
 */
interface EvalCaseResult<I, O> {
  readonly caseId: string;
  readonly input: I;
  readonly output: O;
  /**
   * The dataset's reference answer for this case (`Case.expected`),
   * echoed into the result so persisted reports can be audited without
   * re-joining them against the dataset. Absent when the case declared
   * no expectation.
   */
  readonly expected?: O;
  readonly durationMs: number;
  readonly scores: ReadonlyArray<{
    readonly scorer: string;
    readonly result: ScoreResult;
  }>;
}
/**
 * Final report shape.
 *
 * @stable
 */
interface EvalReport<I, O> {
  readonly results: ReadonlyArray<EvalCaseResult<I, O>>;
  readonly summary: {
    readonly total: number;
    readonly passed: number;
    readonly failed: number;
    readonly avgDurationMs: number;
    readonly byScorer: Readonly<Record<string, {
      readonly passed: number;
      readonly failed: number;
      readonly avgScore: number | null;
    }>>;
    /**
     * 95% Wilson score interval on the overall pass rate.
     * Always present on reports produced by `runEvals`; optional so older
     * persisted reports keep parsing.
     */
    readonly passRateCi?: {
      readonly lo: number;
      readonly hi: number;
    };
    /**
     * pass^k stability metric - fraction of base cases whose EVERY repeat
     * iteration passed. Present only when the run used `iterations > 1`.
     */
    readonly passHatK?: {
      readonly k: number;
      readonly baseCases: number;
      readonly value: number;
    };
  };
  /**
   * `true` when the run was cut short by an aborted signal - `results` and
   * `summary` then cover only the cases that finished before the abort (a
   * partial report). Absent on a normal full run. See `runEvals`.
   */
  readonly aborted?: boolean;
}
/**
 * @stable
 */
interface RunEvalOptions<I, O> {
  readonly agent: {
    readonly run: (input: I) => Promise<O>;
  };
  readonly dataset: Dataset<I, O>;
  readonly scorers: ReadonlyArray<Scorer<I, O>>;
  readonly iterations?: number;
  readonly signal?: AbortSignal;
}
//#endregion
export { Case, Dataset, EvalCaseResult, EvalReport, RunEvalOptions, ScoreResult, Scorer };
//# sourceMappingURL=types.d.ts.map