/** * v1.3.6 §3.3 — skill eval scoring (deterministic bookkeeping). * * The eval *judgments* are made by the running Claude, not an API client: * skill-manager/skill-refinement (Chief in-session) spawns judge sub-agents * via the Task tool — "does this query surface the skill?" (trigger) and * "is the with-skill output better?" (output A/B). This module is only the * deterministic arithmetic around those judgments: tally trigger-rate, split * train/val reproducibly, and aggregate A/B deltas. Pure — no fs here, no LLM. * * See `skills/skill-manager/references/eval-recipe.md` for the procedure the * agent follows; this is the scorer it hands results to. */ export interface TriggerQuery { query: string; /** true = should trigger the skill; false = should-NOT (near-miss negative). */ should: boolean; } export interface TriggerResult extends TriggerQuery { /** Did the skill fire for this query (judged by the running agent)? */ triggered: boolean; } export interface TriggerScore { /** Fraction of should-trigger queries that fired (higher is better). */ shouldRate: number; /** Fraction of should-NOT queries that fired (lower is better). */ shouldNotRate: number; passShould: boolean; passShouldNot: boolean; pass: boolean; counts: { should: number; shouldNot: number; }; } /** * Score one run of trigger results. Pass = should-trigger rate strictly above * `threshold` AND should-NOT rate strictly below it (0.5 default, §3.3). */ export declare function scoreTrigger(results: TriggerResult[], threshold?: number): TriggerScore; /** * Deterministic train/val split (default 60/40, §3.3) so iterations are * reproducible and we never overfit description tweaks to val queries. Uses a * seeded LCG shuffle — same items + seed => same split. */ export declare function splitTrainVal(items: T[], ratio?: number, seed?: number): { train: T[]; val: T[]; }; export interface OutputCaseResult { id: string; withSkillPass: boolean; withoutSkillPass: boolean; tokensWith: number; tokensWithout: number; durationMsWith: number; durationMsWithout: number; } export interface OutputABScore { passRateWith: number; passRateWithout: number; /** with − without; positive means the skill body helped. */ passDelta: number; /** Mean extra tokens the skill costs (with − without). */ tokenDelta: number; /** Mean extra wall-clock the skill costs (with − without). */ durationMsDelta: number; n: number; } /** Aggregate A/B case results into pass-rate + cost deltas (§3.3). */ export declare function scoreOutputAB(results: OutputCaseResult[]): OutputABScore;