import { type HostFingerprint } from "../models/host-state-store.ts"; export type ToolSelectionIntentClass = "read" | "search" | "execute" | "write" | "retrieve" | "explain" | "other"; export interface ToolPerformanceKey { modelRef: string; intentClass: ToolSelectionIntentClass; tool: string; } export interface ToolPerformanceStats extends ToolPerformanceKey { alpha: number; beta: number; sampleCount: number; latencyEwmaMs?: number; latencyDeviationEwmaMs?: number; inputTokenEstimateEwma?: number; outputTokenEstimateEwma?: number; repairCount: number; bounceCount: number; failureCount: number; lastUsedAt: string; } export interface ToolSelectionObservation { at: string; modelRef: string; intentClass: ToolSelectionIntentClass; actualTool: string; firstTool: boolean; succeeded: boolean; disposition: "recommend" | "shortlist" | "abstain"; recommendation?: string; shortlist: string[]; entropy: number; margin: number; /** Redacted ranking: names and numeric scores only, never prompts, arguments, paths, or output. */ ranked: Array<{ tool: string; utility: number; probability: number; }>; latencyMs?: number; inputTokenEstimate?: number; outputTokenEstimate?: number; } export interface ToolSelectionMetrics { firstToolAttempts: number; firstToolSuccesses: number; wrongToolOrFailureCount: number; recommendationCount: number; recommendationMatchedCount: number; shortlistCount: number; abstentionCount: number; averageLatencyMs?: number; averageInputTokenEstimate?: number; averageOutputTokenEstimate?: number; } /** * Durable, per-(model,intent) aggregate of the observe-mode loop: does the raw expected-utility * ranking's top pick (`ToolSelectionObservation.ranked[0]`) match what the model actually called * (`ToolSelectionObservation.actualTool`)? Tracked separately from the capped, rolling * `observations` log so evidence for a given (model,intent) pair survives that log's trimming. * `hintActive*` fields are the SAME agreement measure, but restricted to calls made while an * evidence-gated promotion hint (see promotion.ts) was active for this bucket — the hint's own * efficacy trace: it never gates activation directly (that is always * recomputed live from `ToolPerformanceStats`), but it is the durable evidence a report can show * for "is the hint still earning its keep". */ export interface ToolSelectionIntentAgreement { modelRef: string; intentClass: ToolSelectionIntentClass; sampleCount: number; agreementCount: number; hintActiveSampleCount: number; hintActiveAgreementCount: number; lastUpdatedAt: string; } export interface ToolExecutionObservation { key: ToolPerformanceKey; success: boolean; latencyMs: number; inputTokenEstimate?: number; outputTokenEstimate?: number; selection: Omit; at?: string; /** * Whether an evidence-gated promotion hint (promotion.ts) was already active for this * (model,intent) bucket BEFORE this call was recorded — captured by the controller at * `begin()` time, so it reflects evidence up to but not including this observation. Used only * to bucket the durable agreement stats (`ToolSelectionIntentAgreement.hintActive*`); never * changes what gets recorded, only how it is split for the efficacy report. */ hintActiveAtCallTime?: boolean; } export declare class ToolPerformanceStore { private readonly storage; constructor(filePath: string, options?: { fingerprint?: () => HostFingerprint; readOnly?: boolean; }); static forAgentDir(agentDir: string, options?: { fingerprint?: () => HostFingerprint; readOnly?: boolean; }): ToolPerformanceStore; private createHostData; get(key: ToolPerformanceKey): ToolPerformanceStats; /** One fresh durable snapshot of every per-tool track record for a model. */ getStatsForModel(modelRef: string): ToolPerformanceStats[]; /** Every per-tool track record recorded for a (model,intent) bucket — the promotion.ts input. */ getStatsForIntent(modelRef: string, intentClass: ToolSelectionIntentClass): ToolPerformanceStats[]; /** Durable observe-mode agreement for one (model,intent) bucket (see {@link ToolSelectionIntentAgreement}). */ getIntentAgreement(modelRef: string, intentClass: ToolSelectionIntentClass): ToolSelectionIntentAgreement; /** All recorded (model,intent) agreement buckets, optionally scoped to one model — report input. */ getAllIntentAgreements(modelRef?: string): ToolSelectionIntentAgreement[]; recordValidation(key: ToolPerformanceKey, outcome: "repaired" | "bounced", at?: string): ToolPerformanceStats; recordExecution(observation: ToolExecutionObservation): ToolPerformanceStats; getMetrics(modelRef?: string): ToolSelectionMetrics; getObservations(modelRef?: string): ToolSelectionObservation[]; } //# sourceMappingURL=tool-performance-store.d.ts.map