/**
 * Eval harness: corpus pinning, provenance capture, and run records.
 *
 * The scoring math lives in `eval.ts`; this module is everything around it
 * that makes a number *comparable to a later number*. Three problems it
 * exists to solve, all of which bit the first eval round
 * (docs/hybrid-retrieval-design.md, "Eval results"):
 *
 *  1. **The corpus is the repo.** Retrieval is measured over hoocode itself,
 *     so every commit moves the thing being measured. A baseline taken today
 *     and a rerun taken after a retrieval change differ by both the change
 *     and the intervening commits, and nothing in the output says so. Fix:
 *     run against a detached git worktree pinned to an explicit SHA, and put
 *     that SHA in the record.
 *  2. **Nothing was recorded.** Results were printed to a terminal and
 *     hand-copied into a markdown table with no repo SHA, no embedder
 *     identity, and no index state. Fix: emit a machine-readable run record.
 *  3. **A degraded run looks like a real one.** With no embsearch binary the
 *     semantic and hybrid rows silently degrade to lexical, producing a table
 *     that is all-lexical but reads like a full sweep. Fix: `embedder` in the
 *     record, plus a per-row degraded count that the writer refuses to hide.
 */
import type { EmbsearchService } from "../embsearch/embsearch-service.js";
import { type EvalConfig, type EvalQuery, type EvalQueryResult } from "./eval.js";
/** Metrics aggregated per config across the whole gold set. */
export interface EvalAggregate {
    label: string;
    recallAt1: number;
    recallAt5: number;
    recallAt10: number;
    recallAt50: number;
    mrr: number;
    /** Queries scored under this config. */
    n: number;
    /** How many of them ran degraded (requested retriever unavailable). */
    degraded: number;
}
/** Everything needed to decide whether two run records may be compared. */
export interface EvalProvenance {
    timestampMs: number;
    /** SHA of the corpus actually indexed and searched. */
    corpusSha: string;
    /** Ref the caller asked for, before resolution (e.g. "HEAD"). */
    corpusRef: string;
    /** True when the corpus came from the live working tree rather than a
     *  pinned worktree — results are then not reproducible. */
    corpusFromWorkingTree: boolean;
    /** Uncommitted changes present at run time. Only meaningful (and only
     *  possible) when `corpusFromWorkingTree` is true. */
    corpusDirty: boolean;
    /** Files removed from the corpus before indexing — see
     *  {@link CORPUS_EXCLUSIONS}. A score against a different exclusion list is
     *  a score against a different corpus, so it is recorded, not assumed. */
    corpusExcluded: string[];
    /**
     * Set only when the run scored a deliberately shrunk corpus.
     *
     * A smaller distractor pool makes every query easier, so these metrics are
     * higher than a full-corpus run's and are **not** comparable to one. They
     * are comparable to another subsampled run with the same target and seed,
     * which is what makes this useful for screening model arms.
     */
    corpusSubsample?: CorpusSubsampleInfo;
    /** SHA of the tree whose retrieval code ran. Usually equals `corpusSha`,
     *  but differs when pinning an old corpus with today's code. */
    harnessSha: string;
    /**
     * Content hash of `src/core/search` + the chunker. Every tuning constant
     * that shapes a result — the fusion cap, top-k depths, rerank weights,
     * chunk sizing — lives in those files, so a changed hash means the
     * numbers are not comparable, without this module having to maintain a
     * hand-copied (and inevitably stale) list of constants.
     */
    retrievalSourceHash: string;
    /** Embedding backend state. `available: false` means every semantic and
     *  hybrid row in this record degraded to lexical. */
    embedder: {
        available: boolean;
        reason?: string;
        /** Indexed chunk count when the index reached `ready`. */
        chunkCount?: number;
        phase: string;
        /** Binary that served the embeddings, and its self-reported version. */
        binaryPath?: string;
        binaryVersion?: string;
        /**
         * Model id the daemon reported — the thing that actually identifies
         * which model produced these scores.
         *
         * This used to be inferred from `binaryVersion`, on the reasoning that
         * the model was baked into the binary at build time. `--model <dir>`
         * ends that: one binary now serves any number of models, so two arms of
         * a model comparison would have carried identical provenance and been
         * indistinguishable in the record. The id is a hash over the model's
         * whole spec (pooling, token limit, prefixes), so a change to any of
         * them shows up here.
         */
        modelId?: string;
        /** Model directory passed as `--model`, when the run overrode the
         *  bundled model. Absent means the binary's own model was used. */
        modelDir?: string;
    };
    /** Daemon-side BM25 hybrid store, when the run included one. Absent means
     *  the record has no `daemon-hybrid` rows. */
    daemonHybrid?: {
        available: boolean;
        phase: string;
    };
    /**
     * Wall time, split at the seam between building the index and scoring the
     * gold set.
     *
     * Recorded because the cost side of a model comparison is almost entirely
     * indexing, and a single total cannot show it: the first such comparison
     * could only report "17 -> 60 min" for whole runs and had to note that the
     * figure was "not isolated from query work", which left the headline cost
     * of the change unmeasured. These two numbers are machine- and
     * load-dependent and say nothing about retrieval quality; they are a budget,
     * not a metric.
     */
    /**
     * Chunker character cap, when an arm overrode it. Absent means the shipped
     * `CHUNK_MAX_CHARS`. Records differing here are not comparable: the chunks
     * are different text, so every id, span and vector differs.
     */
    chunkMaxChars?: number;
    timing?: {
        /** Seconds spent bringing the index(es) to `ready`, model load included. */
        indexSeconds: number;
        /** Seconds spent running every config over every gold query. */
        querySeconds: number;
        /** True when one hybrid store served both the dense and BM25 roles
         *  rather than the corpus being embedded twice. Runs with this false
         *  paid roughly double the indexing time. */
        sharedStore: boolean;
    };
    runtime: {
        node: string;
        platform: string;
        arch: string;
    };
}
export interface EvalRunRecord {
    provenance: EvalProvenance;
    goldSet: {
        queryCount: number;
        byClass: Record<string, number>;
        goldSpanCount: number;
    };
    configs: readonly EvalConfig[];
    aggregates: EvalAggregate[];
    perQuery: Array<{
        id: string;
        class: string;
        results: EvalQueryResult[];
    }>;
}
/** Hash every retrieval-shaping source file, so a tuning change is visible as
 *  a changed provenance field rather than an unexplained metric shift. */
export declare function hashRetrievalSource(repoRoot: string): string;
export interface PinnedCorpus {
    /** Directory to index and search. */
    cwd: string;
    sha: string;
    fromWorkingTree: boolean;
    dirty: boolean;
    /** Files removed from the corpus before indexing. Empty when the corpus is
     *  the live working tree, which is never mutated. */
    excluded: string[];
    /** Present only on a subsampled run. Its presence is what marks a record as
     *  incomparable to a full-corpus one. */
    subsample?: CorpusSubsampleInfo;
    /** Removes the worktree, if one was created. */
    dispose: () => void;
}
/** Request to shrink the corpus to a chunk budget. See {@link pinCorpus}. */
export interface CorpusSubsampleRequest {
    /** Approximate chunk budget. Gold-bearing files are kept past it. */
    targetChunks: number;
    /** Files that must survive regardless of budget — the gold-bearing ones. */
    keepRelPaths: readonly string[];
    /** Seed for the distractor draw, so a budget reproduces exactly. */
    seed: number;
}
/** What a subsampled run did, recorded so it can never be read as a full one. */
export interface CorpusSubsampleInfo {
    targetChunks: number;
    /** Chunks actually kept. Exceeds the target when gold files alone do. */
    chunkCount: number;
    filesKept: number;
    filesDropped: number;
    /** Gold-bearing files, all of which are kept unconditionally. */
    goldFilesKept: number;
    seed: number;
}
/**
 * Files that describe this eval rather than being searched by it.
 *
 * The fixtures hold all 62 query strings verbatim, so every query is a perfect
 * lexical match against its own entry, and the design note quotes the same
 * queries while discussing the classes they belong to. Measured before this
 * exclusion existed: **56 of 62 queries had one of these files in the top 10,
 * 27 of 62 had one as the #1 result, and they consumed 133 of the 620
 * top-10 slots** — a fifth of the window, spent on the eval reading itself.
 *
 * That is not a ranking artifact a reranker can fix: it displaces real answers
 * out of the window entirely, which is why two boundary-class queries were
 * absent from the top *50* rather than merely buried. Retrieving your own
 * question is not retrieval, so the corpus is scored without them.
 *
 * Removed from the pinned worktree before indexing, never from the repo — and
 * recorded in the run's provenance so a score is never silently taken against
 * a different corpus than it claims.
 */
export declare const CORPUS_EXCLUSIONS: readonly string[];
/**
 * Materialize the corpus to evaluate.
 *
 * With a `ref`, checks out a detached worktree at that commit so the corpus is
 * byte-identical on every rerun. Without one, falls back to the live working
 * tree and reports `dirty` so the record shows the run was not reproducible.
 *
 * `subsample` shrinks the corpus to a chunk budget, for screening runs where a
 * full arm costs too much to iterate on. It keeps every gold-bearing file and
 * draws distractors deterministically. This is a real change to what is being
 * measured — a smaller distractor pool makes retrieval easier and inflates
 * every metric — so it is recorded in the record and folded into the worktree
 * path, and it needs a `ref`.
 */
export declare function pinCorpus(repoRoot: string, ref: string | undefined, subsample?: CorpusSubsampleRequest): PinnedCorpus;
export declare function collectProvenance(repoRoot: string, corpus: PinnedCorpus, corpusRef: string, service: EmbsearchService | undefined, embsearchBinary?: string, hybridService?: EmbsearchService, modelDir?: string, timing?: EvalProvenance["timing"], chunkMaxChars?: number): EvalProvenance;
export declare function summarizeGoldSet(dataset: readonly EvalQuery[]): EvalRunRecord["goldSet"];
export interface RunEvalSuiteOptions {
    cwd: string;
    dataset: readonly EvalQuery[];
    configs: readonly EvalConfig[];
    service?: EmbsearchService;
    /** Second service backed by a daemon-side BM25 hybrid store, for the
     *  `daemon-hybrid` configs. Absent means those rows are omitted. */
    hybridService?: EmbsearchService;
    onQuery?: (index: number, query: EvalQuery) => void;
}
export declare function runEvalSuite(options: RunEvalSuiteOptions): Promise<{
    aggregates: EvalAggregate[];
    perQuery: EvalRunRecord["perQuery"];
}>;
export declare function formatAggregateTable(aggregates: readonly EvalAggregate[]): string;
//# sourceMappingURL=eval-harness.d.ts.map