/** * Structured failure logging for self-evolution analysis. * Append-only JSONL at ~/.blockrun/failures.jsonl (capped 500 records). * * 2026-05-11: Adopted a Cursor-style tool-failure taxonomy on the * `category` field. Lets us: * 1. Tell at a glance whether a spike of failures is the model's * fault (InvalidArguments), the environment's fault * (UnexpectedEnvironment), an upstream's fault (ProviderError), * a user action (UserAborted), or a slow path (Timeout). * 2. Build per-(tool, category) baselines for anomaly detection — * see `getToolAnomalies()` below. * * The existing single-line errorMessage column is preserved so older * records still parse. classifyToolFailure() auto-classifies records * without a category field on read, so historical entries flow into * the same dashboards without a migration. */ /** * Coarse classification of a tool failure. Mirrors Cursor's published * "Tool reliability" taxonomy so error dashboards translate cleanly * across the industry, but tuned for Franklin's tool surface. */ export type ToolFailureCategory = 'InvalidArguments' | 'UnexpectedEnvironment' | 'ProviderError' | 'UserAborted' | 'Timeout' | 'Unknown'; export interface FailureRecord { timestamp: number; model: string; failureType: 'tool_error' | 'model_error' | 'permission_denied' | 'agent_loop'; toolName?: string; errorMessage: string; recoveryAction?: string; /** * Coarse classification of the failure. Set by recordFailure() when * a record is written, or auto-filled by loadFailures() for older * records that pre-date this field. */ category?: ToolFailureCategory; } /** * Classify a tool failure by matching the error message + tool name * against known patterns. Layered top-to-bottom — first match wins. * `Unknown` is the catch-all; if you see one in production, the * classifier needs a new branch (file a follow-up). */ export declare function classifyToolFailure(errorMessage: string, toolName?: string): ToolFailureCategory; export declare function recordFailure(record: FailureRecord): void; export declare function loadFailures(limit?: number): FailureRecord[]; export declare function getFailureStats(): { byTool: Map; byType: Map; byCategory: Map; total: number; recentFailures: FailureRecord[]; }; export interface AnomalyReport { toolName: string; category: ToolFailureCategory; recentCount: number; baselineCount: number; baselineWindowMs: number; recentWindowMs: number; /** * Multiplier of recent-rate vs baseline-rate. Infinity when the * baseline is zero (i.e. a new failure type appeared). 1.0 = same * rate as baseline. */ spikeRatio: number; /** Most recent error message in this bucket — useful for triage. */ sampleMessage: string; } export interface AnomalyOptions { /** Recent window in ms. Default 24h. */ recentWindowMs?: number; /** Baseline window in ms (counted from now, includes the recent window). Default 30d. */ baselineWindowMs?: number; /** Minimum recent count to consider — filters out single-flake noise. Default 3. */ minRecent?: number; /** Minimum spike ratio to surface. Default 3.0. */ minSpikeRatio?: number; } /** * Compute (tool, category) anomalies vs a rolling baseline. * * Returns the buckets where the recent failure rate is dramatically * higher than baseline — sorted by spike severity. Skips buckets where * `recentCount` is below `minRecent` to avoid surfacing every flaky * one-off. * * A bucket with `baselineCount=0` and `recentCount >= minRecent` is * always surfaced (spikeRatio = Infinity) — these are brand-new failure * modes that the harness has never seen before, and they're the most * important kind to investigate. */ export declare function getToolAnomalies(opts?: AnomalyOptions): AnomalyReport[];