import { type CatalogItem } from './jev.js';
export type EvalKind = 'agents' | 'skills';
/** One task. `agents` / `skills` list EVERY acceptable id; an empty list
 *  means "no expectation" and the task is not scored for that kind. */
export interface EvalTask {
    id: number;
    task: string;
    domain?: string;
    agents: string[];
    skills: string[];
    /** Nothing in the catalogs fits: the gated pick must show nothing. */
    noPick?: boolean;
}
export interface EvalCatalogs {
    agents: CatalogItem[];
    skills: CatalogItem[];
}
export interface EvalMiss {
    id: number;
    task: string;
    expected: string[];
    got: string[];
    /** 1-based position of the first acceptable id within the ranked list, or
     *  null when none was ranked. */
    rank: number | null;
}
export interface KindScore {
    /** Tasks scored for this kind (those with at least one expected id). */
    n: number;
    top1: number;
    top3: number;
    /** Tasks whose top-1 was not acceptable. */
    misses: EvalMiss[];
}
export interface EvalReport {
    tasks: number;
    agents: KindScore;
    skills: KindScore;
}
/** Ranked ids per kind for one task (at least the top 3). */
export interface TaskPicks {
    agents: string[];
    skills: string[];
}
export declare const EVAL_TOP = 3;
export declare function evalDir(root: string): string;
/** dataset.json + holdout.json under tests/pick-eval, or null outside a
 *  checkout that carries the eval set. */
export declare function readEvalTasks(root: string, sets?: string[]): EvalTask[] | null;
/** The frozen catalog snapshot (catalog-snapshot.json), or null. */
export declare function readEvalSnapshot(root: string): EvalCatalogs | null;
/**
 * The project's live catalogs as the picker sees them, minus the user's
 * personal ~/.claude/skills so a score means the same on every machine.
 */
export declare function projectCatalogs(root: string): EvalCatalogs;
/** Score ranked picks against the tasks' acceptable ids. */
export declare function scorePicks(tasks: EvalTask[], picks: TaskPicks[]): EvalReport;
/** Keyword-only picks for every task (in-process, no network). */
export declare function keywordPicks(tasks: EvalTask[], catalogs: EvalCatalogs): TaskPicks[];
export declare function keywordEval(tasks: EvalTask[], catalogs: EvalCatalogs): EvalReport;
/** Expected ids a catalog does not contain (a stale expectation). */
export declare function unknownExpectations(tasks: EvalTask[], catalogs: EvalCatalogs): {
    id: number;
    kind: EvalKind;
    missing: string[];
}[];
export declare function formatScore(s: KindScore): string;
/** What the gated pick (the [PICK] bar: pick-rank.cjs KEYWORD_GATE) showed
 *  for one kind. `precision` = correct / shown; a show on a `noPick` task, or
 *  of an id the task does not accept, is wrong. */
export interface GatedScore {
    /** Tasks judged: those with an expectation for this kind, plus noPick tasks. */
    n: number;
    shown: number;
    correct: number;
    precision: number | null;
    wrong: {
        id: number;
        task: string;
        shown: string;
    }[];
}
/** Keyword picks through the prompt hook's gate: how often it shows one, and
 *  how often what it shows is right. */
export declare function gatedEval(tasks: EvalTask[], catalogs: EvalCatalogs): {
    agents: GatedScore;
    skills: GatedScore;
};
//# sourceMappingURL=pick-eval.d.ts.map