/**
 * The System One benchmark runner (T12495): replay every labelled row through
 * every compared provider with `decideBatch`, next to each site's heuristic,
 * and score the answers.
 *
 * - Provider answer cache off (`cache: false` on every request) and the
 *   in-process outcome cache off, so every row is a real provider call.
 * - One batch deadline of at least {@link MIN_BENCH_TIMEOUT_MS} (spec §1: the
 *   batch endpoint answers its items serially).
 * - A hard total spend cap across all providers and runs, SEPARATE from the
 *   everyday monthly cap: before each batch the spend estimate (per
 *   question, the larger of {@link DECISION_COST_ESTIMATE_MICROS_PER_QUESTION}
 *   and the connection's observed mean reported cost) is added to what was
 *   spent so far, and the run stops cleanly when that would pass the cap.
 *   Billed answers the client rejected (`invalid_response`) count at their
 *   reported cost (else the estimate), so a model that keeps answering in
 *   the wrong shape is still capped. The monthly cap does not gate the benchmark
 *   (`spend: null` on the calls), but every batch's spend is still recorded
 *   in the monthly ledger so month-to-date spend stays true.
 * - Providers take turns batch by batch, so a cap stop leaves them with the
 *   same coverage.
 *
 * @task T12495
 * @epic T12486
 */
import { type SpendLedger } from '../spend.js';
import type { DecideFetch } from '../transport.js';
import { type BenchDatasetSizes } from './dataset.js';
import { type BenchClassificationMetrics, type BenchLatency } from './metrics.js';
import { type BenchSpread } from './stats.js';
import { type BenchConnection, type BenchRow, type BenchSite } from './types.js';
/** Default hard spend cap across providers and runs: $5. */
export declare const DEFAULT_BENCH_MAX_MICROS = 5000000;
/** Minimum (and default) batch deadline, ms. */
export declare const MIN_BENCH_TIMEOUT_MS = 30000;
/** Default rows per batch call. */
export declare const DEFAULT_BENCH_BATCH_SIZE = 16;
/** Largest batch the layahost batch endpoint accepts (64 requests). */
export declare const MAX_BENCH_BATCH_SIZE = 64;
/** Name of the heuristic pseudo-provider in results. */
export declare const HEURISTIC_PROVIDER = "heuristic";
/** Options for {@link runDecideBench}. */
export interface RunDecideBenchOptions {
    /** Labelled rows (already filtered to the sites to run). */
    readonly rows: readonly BenchRow[];
    /** Providers to compare. */
    readonly connections: readonly BenchConnection[];
    /** Repeated runs. Default 1. */
    readonly runs?: number;
    /** Hard total spend cap in micro-dollars. Default {@link DEFAULT_BENCH_MAX_MICROS}. */
    readonly maxMicros?: number;
    /** Rows per batch call, 1–{@link MAX_BENCH_BATCH_SIZE}. Default {@link DEFAULT_BENCH_BATCH_SIZE}. */
    readonly batchSize?: number;
    /** Batch deadline, ms; raised to at least {@link MIN_BENCH_TIMEOUT_MS}. */
    readonly timeoutMs?: number;
    /** Monthly ledger the actual spend is recorded in; `null` skips it. Default: the file ledger. */
    readonly spendLedger?: SpendLedger | null;
    /** Transport (tests). Default: the decide transport. */
    readonly fetch?: DecideFetch;
    /**
     * Directory for per-connection provider-state files (detected
     * capabilities). Default `<cleoHome>/decide/bench/`. Each compared
     * connection gets its own file: the everyday state file holds ONE identity,
     * so two connections sharing it would overwrite each other's detection and
     * one would fall back to the Jev minimum (no batch, no `cache: false`).
     */
    readonly providerStateDir?: string;
    /** Clock (tests). */
    readonly now?: () => Date;
}
/** One provider's (or the heuristic's) result on one site in one run. */
export interface BenchSiteResult extends BenchClassificationMetrics {
    /** Site. */
    readonly site: BenchSite;
    /** Rows sent (or scored, for the heuristic). */
    readonly attempted: number;
    /** Rows the provider answered usably (the metrics' `n`). */
    readonly answered: number;
    /** Latency of provider-answered rows: the wall time of the batch call that carried each row. */
    readonly latency: BenchLatency;
    /** Spend accounted to this site, micro-dollars (reported cost, else the estimate). */
    readonly costMicros: number;
    /** Whether the provider reported its cost (`meta.cost_micros` / `cost_usd`). */
    readonly costReported: boolean;
    /** Fallback reason → rows (timeouts, 4xx/5xx, malformed answers, …). */
    readonly fallbacks: Readonly<Record<string, number>>;
    /** Rows the provider answered with a value outside the site's labels. */
    readonly unusable: number;
}
/** One provider in one run. */
export interface BenchProviderRun {
    /** Profile name. */
    readonly provider: string;
    /** Per-site results, in {@link BENCH_SITES} order. */
    readonly sites: readonly BenchSiteResult[];
}
/** One repeated run. */
export interface BenchRunRecord {
    /** 1-based run number. */
    readonly run: number;
    /** Whether every batch of the run was sent (false when the cap stopped it). */
    readonly complete: boolean;
    /** Per provider. */
    readonly providers: readonly BenchProviderRun[];
}
/** Mean and spread of one provider × site across runs. */
export interface BenchAggregateRow {
    /** Profile name (or {@link HEURISTIC_PROVIDER}). */
    readonly provider: string;
    /** Site. */
    readonly site: BenchSite;
    /** Rows answered, per run. */
    readonly answered: BenchSpread;
    /** Accuracy. */
    readonly accuracy: BenchSpread;
    /** Precision. */
    readonly precision: BenchSpread;
    /** Recall. */
    readonly recall: BenchSpread;
    /** F1. */
    readonly f1: BenchSpread;
    /** False-positive rate. */
    readonly falsePositiveRate: BenchSpread;
    /** p50 latency, ms. */
    readonly p50Ms: BenchSpread;
    /** p95 latency, ms. */
    readonly p95Ms: BenchSpread;
    /** Cost per run, micro-dollars (reported cost, else the estimate). */
    readonly costMicros: BenchSpread;
    /**
     * Whether any run's provider reported its cost. False for a plain Jev host
     * (no `meta.cost_*`): `costMicros` is then only the reservation estimate.
     */
    readonly costReported: boolean;
    /** Fallbacks summed over runs, by reason. */
    readonly fallbacks: Readonly<Record<string, number>>;
}
/** Spend accounting of a benchmark. */
export interface BenchSpendSummary {
    /** The hard cap, micro-dollars. */
    readonly capMicros: number;
    /** Spent (reported cost, else the estimate), micro-dollars. */
    readonly spentMicros: number;
    /** Of which the providers reported. */
    readonly reportedMicros: number;
    /** Whether the cap stopped the benchmark. */
    readonly capReached: boolean;
    /** Where it stopped, when it did. */
    readonly stoppedAt?: {
        readonly run: number;
        readonly site: BenchSite;
        readonly provider: string;
        readonly batch: number;
        readonly estimateMicros: number;
    };
    /** Batch calls made. */
    readonly batchesSent: number;
    /** Batch calls planned (runs × sites × batches × providers). */
    readonly batchesPlanned: number;
    /** Whether the monthly ledger accepted the spend record (false: it could not be written). */
    readonly recordedInMonthlyLedger: boolean;
}
/** The machine-readable benchmark result. */
export interface BenchResults {
    /** Format version. */
    readonly schemaVersion: 1;
    /** Owning task. */
    readonly task: 'T12495';
    /** ISO start time. */
    readonly startedAt: string;
    /** ISO end time. */
    readonly finishedAt: string;
    /** Run configuration; never contains an API key. */
    readonly config: {
        readonly runs: number;
        readonly batchSize: number;
        readonly timeoutMs: number;
        readonly maxMicros: number;
        readonly sites: readonly BenchSite[];
        readonly providers: readonly {
            readonly name: string;
            readonly provider: string;
            readonly baseUrl: string;
            readonly model?: string;
        }[];
    };
    /** Dataset sizes. */
    readonly dataset: BenchDatasetSizes;
    /** The heuristic baseline (deterministic, scored once). */
    readonly heuristic: readonly BenchSiteResult[];
    /** Every run. */
    readonly runs: readonly BenchRunRecord[];
    /** Mean and spread per provider × site (heuristic included). */
    readonly aggregate: readonly BenchAggregateRow[];
    /** Spend. */
    readonly spend: BenchSpendSummary;
}
/**
 * Run the benchmark.
 *
 * Never throws because of a provider: failures become per-row fallbacks with
 * their reason. Stops cleanly at the spend cap and reports what it completed.
 *
 * @param opts - Rows, connections, runs, cap and wiring.
 * @returns The results.
 */
export declare function runDecideBench(opts: RunDecideBenchOptions): Promise<BenchResults>;
//# sourceMappingURL=runner.d.ts.map