/** * Grounding evaluator — a cheap second-pass check that every factual claim * in Franklin's answer traces back to a tool-call result, not model memory. * * Why this exists (2026-04 retrospective): the CRCL incident — user asked * about a stock Franklin had tools to query, Franklin answered from 2022 * training data instead. Root cause wasn't a prompt defect; it was an * absent evaluator. The existing `verification.ts` only fires when the * agent writes code (Edit / Write / Bash threshold), so read-heavy hero * use cases (trading, research, analysis) never triggered any quality gate. * * This module is the complement: fires on *answers with factual content*, * regardless of tool type. Anthropic's harness-design article calls out * "self-evaluation on complex tasks" as anti-pattern #14 — models skew * positive when grading themselves. So the check runs as a separate agent * (different system prompt, explicitly adversarial) with its own model. * * v1 scope: check only, never re-prompt. Emit a follow-up ⚠️ event when * claims look ungrounded, let the user decide whether to re-ask. The * re-prompt loop (generator iterates against evaluator findings until * PASS) is a v2 concern once we know v1 catches real cases without * false-positive noise. */ import type { CapabilityHandler, Dialogue } from './types.js'; import { ModelClient } from './llm.js'; export type GroundingVerdict = 'GROUNDED' | 'PARTIAL' | 'UNGROUNDED' | 'SKIPPED'; export interface GroundingResult { verdict: GroundingVerdict; issues: string[]; raw: string; } /** * Decide whether this turn warrants a grounding check. Principles: * - Non-trivial user input (not a greeting, not a slash command), OR * the assistant answer contains specific factual claims (numbers + units, * currency, dates, times) regardless of input length * - Non-trivial assistant text output (not just a tool-result echo) * * Intentionally NOT gating on tool-type (read vs write) — the whole point * of this module is to cover read-heavy turns the code verifier misses. */ export declare function shouldCheckGrounding(userInput: string, assistantText: string): boolean; /** * Find the `[FRANKLIN HARNESS PREFETCH]` block in the most recent user * message (that's where intent-prefetch injects it). Returns the inner * payload or null if no prefetch happened this turn. */ export declare function extractPrefetchBlock(history: Dialogue[]): string | null; export declare function parseGroundingResponse(raw: string): GroundingResult; /** Cheap model for grading. Default matches existing verification.ts * choice so both quality gates have the same cost profile. Override via * `FRANKLIN_EVALUATOR_MODEL` to experiment with accuracy/cost trade-offs. */ export declare function evaluatorModel(): string; export declare function checkGrounding(userInput: string, history: Dialogue[], assistantText: string, client: ModelClient, opts?: { abortSignal?: AbortSignal; model?: string; }): Promise; /** * Convert a grounding result into a user-facing follow-up message. Returns * empty string when verdict is GROUNDED / SKIPPED — no reason to spam the * user when the check agreed the answer was sound. */ export declare function renderGroundingFollowup(result: GroundingResult): string; /** * Build a synthetic user message that instructs the agent to retry with the * missing tools. Returned message goes into history so the model's next * generation sees it as the most recent instruction. This is the GAN-like * feedback loop pattern from Anthropic's harness-design writeup — * evaluator findings feed back into the generator until PASS (or retry cap). * * Intentionally terse: the agent already has the original question in * history; we only need to name the gap + the tools to use. */ /** * Pull the tool names the evaluator suggested out of its issue lines. * Issue lines look like: * Claim: "..." → missing tool: WebSearch * Refusal: "..." → should have called: TradingMarket * ... → missing tool: WebSearch (or any distance calculation tool) * * Returns first-token-of-each-comma/pipe-segment names, deduplicated. * Used by both the retry instruction (to name them in prose) and the * loop's tool_choice selection (to pin the next request to a tool). */ export declare function extractMissingToolNames(result: GroundingResult): string[]; export declare function buildGroundingRetryInstruction(result: GroundingResult, originalUserQuestion: string): string; export type { CapabilityHandler };