/** * Knowledge gap signal analyzer. * * Implements the v0.1 detection pipeline defined in * docs/specs/knowledge-gap-signal-spec.md: * * gap_rate = (samples with ≥1 gap signal) / (successfully-executed samples) * * Four signal types are extracted per sample: * 1. failed_search — Grep/Read/Bash search that returned empty or failed * 2. explicit_marker — 【推断】【知识缺口】【未知】 markers in agent output * 3. hedging — hedging language in agent output (accepts FP in v0.1) * 4. repeated_failure — ≥3 consecutive failed searches of the same tool type * * Intentionally does NOT touch report rendering, trends, or CI — those are * step-3 work. This module only computes the structured GapReport. */ import type { ExecutorFn, GapReport, GapSignalRef, ResultEntry, ToolCallInfo, TurnInfo, VariantResult } from '../types/index.js'; import { type ClassifyOptions } from './hedging-classifier.js'; export type GapSignalType = GapSignalRef['type']; export type GapSignal = GapSignalRef; /** * v0.2 severity weights (spec §6). * * Strong (1.0): 硬证据类——agent 工具层真的撞墙了 * - failed_search: Grep/Read 返回空或失败,确定性 miss * - repeated_failure: 同一类查询连续 ≥3 次失败,已不是偶然 * * Weak (0.5): 自我陈述类——依赖模型配合,可能假阳 * - explicit_marker: 依赖 agent 按约定打【推断】等标记,可能漏标 * - hedging: 字符串匹配 "我不确定/可能是"等,假阳率已知较高(spec §2.8) * * Aggregation: 每个用例取其信号的最高权重作"用例严重度",平均到 weightedGapRate。 * weightedGapRate ≤ gapRate,差值反映"软信号占比"——读者据此判断结果可信度。 */ export declare const SIGNAL_WEIGHTS: Record; /** * Decide whether a tool call counts as a "failed search" signal. * * Covers: * - Read with success: false * - Grep with success: false * - Grep with success but empty/"No matches found" output (still a miss) * - Bash with grep/rg/find or explicit probe commands that returned empty or failed */ export declare function isFailedSearchTool(tc: ToolCallInfo): boolean; /** * Extract failed-search signals from a flat tool call sequence. * Deduplicates consecutive duplicates (same tool + same pattern) so an agent * that retries the exact same Grep 5 times counts as one signal, not five. */ export declare function extractFailedSearchSignals(toolCalls: ToolCallInfo[]): GapSignal[]; export declare function collectAssistantTextFromTurns(turns: TurnInfo[] | undefined): string; /** * Find explicit "I'm inferring / I don't know" markers in agent output text. * Each marker occurrence produces one signal; dedupe is at the sample level * (handled by the aggregator, not here). */ export declare function extractMarkerSignals(text: string): GapSignal[]; /** * Match hedging-language phrases in agent output text. * v0.1 emits at most one hedging signal per sample — the aggregator upgrades * this from "per pattern" to "per sample", since multiple hedges in the same * turn are usually coming from the same epistemic state. */ export declare function extractHedgingSignals(text: string): GapSignal[]; /** * Detect ≥N consecutive failed searches of the same tool type within a turn * sequence. Each qualifying run emits exactly one signal anchored at the * first failure in the run. */ export declare function extractRepeatedFailureSignals(turns: TurnInfo[] | undefined): GapSignal[]; /** * Run all four signal extractors on a single variant's result for one sample. * Output signals carry the provided sampleId. */ export declare function extractGapSignalsFromSample(variantResult: VariantResult, sampleId: string): GapSignal[]; export declare function extractGapSignalsFromTrace(input: { sampleId: string; turns?: TurnInfo[]; toolCalls?: ToolCallInfo[]; assistantText?: string; }): GapSignal[]; /** * Compute the gap report for one variant across all samples in a report. * * Denominator rule (spec §5): samples with `ok: false` (the variant completely * failed to execute) are EXCLUDED from both numerator and denominator. * gap_rate is computed only on samples that actually produced a trace. */ export declare function computeGapReport(results: ResultEntry[], variant: string): GapReport; /** * Compute gap reports for every variant in a Report. Convenience wrapper. */ export declare function computeReportGapRates(results: ResultEntry[], variants: string[]): Record; /** * 对一份 GapReport 做 hedging classifier 二次过滤(spec §四.3, v0.2)。 * 流程: * 1) 收集 signals 中所有 type='hedging' 的 candidate * 2) 调 classifyHedgingCandidates 拿 verdict * 3) verdict.isUncertainty=false 的 signal 丢弃 * 4) verdict.isUncertainty=true 的保留并挂 classifierVerdict * 5) recompute samplesWithGap / gapRate / weightedGapRate / byType * * 失败降级在 classifier 层:单批失败 → 该 batch 全部 isUncertainty=true (保守保留)。 * 调用方拿到的 report 永远是合法的,不会因 classifier 失败而崩。 */ export declare function applyHedgingClassifier(report: GapReport, executor: ExecutorFn, opts: ClassifyOptions): Promise<{ report: GapReport; costUSD: number; truncated: boolean; }>; /** * 批量为一组 variant 的 gap reports 跑 classifier。 * 串行(不并行)避免 rate limit + 让 cache hit 在第一批后被后续 batch 复用。 */ export declare function applyHedgingClassifierToReports(reports: Record, executor: ExecutorFn, opts: ClassifyOptions): Promise<{ reports: Record; costUSD: number; }>;