import type { EvalVerdict } from './evaluate.js'; import type { LlmProvider, TokenCounter } from './types.js'; /** * Which few-shot examples earn their tokens, measured rather than guessed. * * The `redundant-examples` advisory answers a *textual* question: does this * example look like an earlier one? That catches the way few-shot blocks actually * grow — copy the last one, change two fields — and it is the cheapest useful * thing to say, because it costs nothing. * * This answers a different and much stronger question: **does removing this * example change any answer?** Two examples can be textually unalike and teach the * same thing, and a block nobody has measured is usually where the tokens are: a * few-shot section is routinely most of a prompt. * * The method is leave-one-out against the prompt's own noise floor. Ask the full * prompt twice to find out how much the model disagrees with *itself*, then remove * one example and ask again. If the answer moves no further than the model already * moves on its own, that example is not doing observable work. * * **It spends the caller's money and must never run by default.** The bill is * `2 + examples` calls per input, and `plannedCalls` exists so a caller can print * the figure before deciding rather than discovering it afterwards. * * **What it cannot tell you.** An example may exist for a case these inputs do not * contain — the boundary condition somebody hit in production last March and added * a demonstration for. Removing it would change nothing measurable here and break * that case. So this reports "no effect on these inputs", never "delete this", and * the wording is deliberate: the strength of the claim is bounded by the inputs * given, and only the caller knows whether those cover what matters. */ export interface ExampleContribution { /** Position in the prompt's example block, from zero. */ index: number; /** The example itself, so a report can quote its first line. */ text: string; tokens: number; /** * Mean agreement between the full prompt's answer and the answer with this * example removed, across every input. */ agreementWithout: number; /** * `indistinguishable` and `within-noise` both mean the removal changed nothing * this measurement can see. `diverges` means it did. `inconclusive` means the * model disagreed with itself too much for any of this to mean anything. */ verdict: EvalVerdict; } export interface PruneReport { provider: string; model: string; /** The model's agreement with itself, given the full prompt. The yardstick. */ selfAgreement: number; contributions: ExampleContribution[]; /** Tokens held by examples whose removal changed nothing observable. */ recoverableTokens: number; /** Calls actually made, so the bill is never a surprise. */ callsMade: number; } export interface PruneOptions { concurrency?: number; countTokens?: TokenCounter; } /** * What this will cost, before it costs it. * * Pure and exported so a CLI can print the number and let somebody say no. A * feature that spends money and only reports the total afterwards is a feature * people run once. */ export declare function plannedCalls(examples: number, inputs: number): number; /** * The prompt with one example block removed, located by position rather than by * text. * * `prompt.replace(block.text, '')` would be shorter and wrong: two identical * example blocks — which is exactly what a copy-paste few-shot section contains — * would both match the first occurrence, so removing the second would silently * remove the first and the measurement would describe a prompt nobody asked about. * Scanning forward from the end of the previous block gives each block its true * offset. */ export declare function withoutExample(prompt: string, examples: readonly { text: string; }[], index: number): string; /** Thrown rather than returned: there is nothing to report and a reason to say. */ export declare class NothingToPrune extends Error { } export declare function pruneExamples(prompt: string, inputs: readonly string[], provider: LlmProvider, options?: PruneOptions): Promise; //# sourceMappingURL=prune.d.ts.map