/** * toolChoiceRecorder — runtime tool-choice margins (RFC-002 tier 2, * blocks C4–C6). * * Per LLM call that OFFERED tools, this recorder captures the menu the * model saw (`stream.llm_start.tools`), what it actually invoked * (`stream.tool_start`), and the choice context — then, LAZILY on first * read, ranks the offered candidates against that context via * influence-core's `scoreMargin` (C4): * * margin = score(best chosen) − score(best non-chosen) * * Small margin (`narrow`, < `marginThreshold`, default 0.05) = the * choice was a close call under the proxy. Top-scored candidate not * among the chosen (`proxyDisagreement`) is ALWAYS flagged — either a * proxy miss or a genuinely surprising model choice; both are exactly * what a debugger wants surfaced. * * ## The choice context (C4 — what is embedded, precisely) * * `buildChoiceContext` assembles the SAME two slots the model's * tool-selection reasoning ran on: * * INCLUDED * 1. the user message of the current turn (`agent.turn_start.userPrompt`) * — the task the model is choosing a tool FOR (first * `maxSlotChars` chars: the head states the task); * 2. the latest assistant reasoning text — the most recent * `stream.llm_end.content` of this turn, when present (last * `maxSlotChars` chars: the tail is where "what next" lives). * Iteration 1 has no assistant text; the slot is omitted. * * EXCLUDED (deliberately) * - the system prompt: constant across every call of the run — zero * per-call discrimination, it only dilutes the embedding; * - older history turns: recency dominates tool choice, and the full * transcript grows the embedding cost linearly with run length; * - raw tool results: the model reads them, but their distilled * effect on the NEXT choice is the assistant's own reasoning text, * which IS included; raw payloads skew the embedding toward data * vocabulary (the honest-proxy discipline: mirror the * decision-relevant text, not every visible byte); * - tool schemas: those are the CANDIDATES being ranked, not context. * * Candidate text per offered tool = `confusabilityText` (tokenized name * + description) — the SAME construction the catalog lint (C1) embeds, * so build-time confusability and runtime margins measure one geometry. * * ## Laziness (C5) * * Event hooks only RECORD (string copies into a KeyedStore). The * embedder runs on the first `getCalls()` / `getFlagged()` / * `getSummary()` — embedding I/O NEVER rides the hot path, even when * the recorder is attached inline. Scores memoize per entry. Attach * with `{ delivery: 'deferred' }` (footprintjs RFC-001) to move the * bookkeeping off the hot path too — it is a normal CombinedRecorder. * * Pattern: CombinedRecorder (Convention 1 — single purpose: tool-choice * margin evidence). Owns a `KeyedStore` keyed * by the LLM call's `runtimeStageId`. Convention 4: resets on a * new `runId` via `FlowRecorder.onRunStart`. * Role: Tier-3 /observe recorder. Attach via `Agent.create(...) * .recorder(handle)` or `executor.attachCombinedRecorder`. * * Honest claim (RFC-002 §2): margins are embedding geometry between the * context and tool descriptions — a deterministic PROXY for the model's * selection function, never "the model chose because". Tier 3 * (choice-entropy sampling) validates the proxy. */ import type { EmitEvent } from 'footprintjs'; import { type Embedder, type MarginResult } from '../../lib/influence-core/index.js'; /** Minimal structural slice of footprintjs's run-boundary events — the * `FlowRunEvent` (`onRunStart`/`onRunEnd`/`onRunFailed`). runId is all we * read (Convention 4). */ interface RunBoundaryEvent { readonly traversalContext?: { readonly runId?: string; }; } /** Minimal structural slice of footprintjs's `onResume` union — on * `CombinedRecorder`, `onResume` is one of the SharedLifecycleOverlap * methods (declared on both `ScopeRecorder` and `FlowRecorder` with * DIFFERENT payloads), so its type is `ResumeEvent | FlowResumeEvent`, not * a single `FlowRunEvent`-shaped type like the three siblings above. Only * the control-flow `FlowResumeEvent` carries `traversalContext`; the * scope-channel `ResumeEvent` (this recorder also receives it, via * `onEmit`) has none — `traversalContext?.runId` is `undefined` there and * the handler ignores it. * * `hasInput` is listed here too, even though the handler never reads it. * Both union members declare `hasInput: boolean`, and TypeScript's * weak-type check rejects an assignment to an object type whose * properties are ALL optional when the source has NO property name in * common with it at all. A slice with only `traversalContext` shares zero * property names with the scope-channel `ResumeEvent` (it has no * `traversalContext` field), so TypeScript flagged the whole handle as not * assignable to `CombinedRecorder` (the 7.3.0 regression: f0a87de added * `onResume` typed with exactly that too-narrow slice). Declaring * `hasInput` gives this slice at least one property name in common with * BOTH union members, which is all the weak-type check requires — the * field carries no behavior. */ interface ResumeBoundaryEvent { readonly hasInput?: boolean; readonly traversalContext?: { readonly runId?: string; }; } /** One offered tool, as the model saw it on `llm_start`. */ export interface OfferedTool { readonly name: string; readonly description?: string; } /** Why an entry has no margin. */ export type ToolChoiceSkipReason = /** The model answered without invoking any tool — no choice to score. */ 'nothing-chosen' /** A chosen tool name was not in the offered catalog (wiring anomaly — * surfaced, not silently massaged). */ | 'chosen-not-offered'; /** One LLM call that offered tools — the recorder's public row shape. */ export interface ToolChoiceCall { /** runtimeStageId of the LLM-call stage (the KeyedStore key). */ readonly runtimeStageId: string; readonly iteration: number; /** The catalog the model saw, in request order. */ readonly offered: readonly OfferedTool[]; /** Unique tool names actually invoked, in first-call order. */ readonly chosen: readonly string[]; /** Every invocation (parallel calls + repeat calls of one tool visible). */ readonly toolCallIds: readonly string[]; /** The choice context that was (or will be) embedded. */ readonly contextText: string; /** Ranked scores + margin + flags. Undefined until scored, or when * `skipped` says why it never will be. */ readonly margin?: MarginResult; readonly skipped?: ToolChoiceSkipReason; } /** Run-summary counts (C6). */ export interface ToolChoiceSummary { /** LLM calls that offered tools (= recorded entries). */ readonly llmCallsWithTools: number; /** Entries where the model invoked at least one tool. */ readonly choices: number; /** Entries with a computed `margin`. */ readonly scored: number; /** narrow OR proxy-disagreement. */ readonly flagged: number; readonly narrow: number; readonly proxyDisagreement: number; /** Entries that will never score (`skipped` set). */ readonly skipped: number; } export interface ToolChoiceRecorderOptions { /** * Injected embedder — runs ONLY on first read (lazy). Wrap in * `embeddingCache(...)` (agentfootprint/observe) so repeated tool * descriptions embed once across calls and runs. */ readonly embedder: Embedder; /** Margins below this flag `narrow`. Default 0.05 (RFC-002 §4). */ readonly marginThreshold?: number; /** Per-slot cap for the choice context. Default 2000 chars. */ readonly maxSlotChars?: number; /** Recorder id (default 'tool-choice'). */ readonly id?: string; } export interface ToolChoiceRecorderHandle { readonly id: string; /** All recorded LLM calls (scored on demand — first call runs the * embedder; results memoize). Entries still open mid-run stay * unscored until they close. */ getCalls(): Promise; /** Calls whose choice was fragile: `narrow` margin OR * `proxyDisagreement` (always flagged) — C6. */ getFlagged(): Promise; /** Run-summary counts (C6). Scores pending entries first. */ getSummary(): Promise; clear(): void; onEmit(event: EmitEvent): void; onRunStart(event: RunBoundaryEvent): void; onResume(event: ResumeBoundaryEvent): void; onRunEnd(event: RunBoundaryEvent): void; onRunFailed(event: RunBoundaryEvent): void; } /** C4: the precise choice-context construction (see module JSDoc for the * include/exclude rationale). Exported so consumers can reproduce it. */ export declare function buildChoiceContext(args: { readonly userPrompt: string; readonly latestAssistantText?: string; readonly maxSlotChars?: number; }): string; /** Build the tool-choice margin recorder (C5). */ export declare function toolChoiceRecorder(options: ToolChoiceRecorderOptions): ToolChoiceRecorderHandle; export {};