/** * Feature extraction: the cheap signals that drive classification. * * Everything here is a pure function of the normalized request. No tokenizer, * no I/O, no model call — this runs on every agent turn, so the work is a few * linear scans over the message list and at most one pass over the newest * user-authored text. */ import type { NormMessage, NormRequest } from "../wire/types.ts"; import type { Features, PromptAnatomy } from "./types.ts"; /** * Complexity signals. Deliberately small: each hit pushes the turn toward a * more expensive tier, so precision beats recall. Word-boundary matches, * evaluated only against the newest user-authored content — never history. */ const COMPLEXITY_KEYWORDS: ReadonlyArray = [ ["architecture", /\barchitecture\b/i], ["refactor", /\brefactor\w*/i], ["debug", /\bdebug\w*/i], // "race" (condition); inflected forms are too rare in agent traffic to matter. ["race", /\brace\b/i], ["deadlock", /\bdeadlock\w*/i], // Causal "why" questions demand reasoning, not retrieval. ["why", /\bwhy\b/i], ["root cause", /\broot cause\b/i], ["design", /\bdesign\w*/i], ["optimize", /\boptimi[sz]\w*/i], ["security", /\bsecurity\b/i], ["migrate", /\bmigrat\w*/i], ["concurrency", /\bconcurren\w*/i], ["invariant", /\binvariant\w*/i], ["proof", /\bproof\w*/i], ]; /** Triviality signals: mechanical edits a small model cannot fumble. */ const TRIVIALITY_KEYWORDS: ReadonlyArray = [ ["rename", /\brename\w*/i], ["typo", /\btypos?\b/i], ["format", /\bformat\w*/i], // Dependency/version bumps. ["bump", /\bbump\w*/i], ["comment", /\bcomments?\b/i], ["changelog", /\bchangelogs?\b/i], ["add a test", /\badd (?:a|an|some) tests?\b/i], ["lint", /\blint(?:ing|ed)?\b/i], ]; /** * Error markers scanned in tail tool results. Kept short and literal on * purpose: a false positive escalates a turn that was actually fine, which * costs real money; a false negative merely routes like a clean continuation. */ const TOOL_FAILURE_MARKERS: ReadonlyArray = [ // "error:"/"Error:" at a line start — the near-universal tool error prefix. /(?:^|\n)\s*(?:error|Error):/, // Python crash dump. /Traceback \(most recent call last\)/, // Shell: missing binary. /command not found/, // Non-zero process exit. "exit code 0" is success and never matches. /exit(?:ed with)? code [1-9]\d*/i, // GNU make: "make: *** [target] Error 2". "Error" sits mid-line so the // prefix rule above misses it, and the "*** " literal is distinctive // enough that successful builds never produce it. /(?:^|\n)[^\n]*\*\*\* \[[^\]]*\] Error \d+/, ]; // Unified-diff headers. Plain "--- "/"+++ " are excluded: markdown rules and // lists would false-positive. `diff --git`, hunks, and a/ b/ paths are real diffs. const DIFF_RE = /(?:^|\n)(?:diff --git |@@ -\d|\+{3} b\/|-{3} a\/)/; // A "terse instruction" is one short sentence; longer text carries real requirements. const TERSE_MAX_BYTES = 128; /** Index after the last message of the trailing run matching `pred`. */ function trailingRunStart(messages: NormMessage[], pred: (m: NormMessage) => boolean): number { let i = messages.length - 1; while (i >= 0) { const m = messages[i]; if (m === undefined || !pred(m)) break; i--; } return i + 1; } export function extractFeatures(req: NormRequest, promptTokens: number): Features { const messages = req.messages; const tail = messages[messages.length - 1]; // The volatile tail: either the newest user-authored content (a trailing // run of user messages — what the human just supplied) or, when the tail is // tool output, a mechanical agent-loop continuation with no new user content. const isToolResultContinuation = tail?.role === "tool"; let newContentBytes = 0; let newestUserText = ""; // Images in the volatile tail: an image the human just supplied is visual // work; a tool-result continuation carries none of its own. let newestRunImages = 0; if (isToolResultContinuation) { const start = trailingRunStart(messages, (m) => m.role === "tool"); for (let i = start; i < messages.length; i++) newContentBytes += messages[i]?.textBytes ?? 0; } else if (tail?.role === "user") { const start = trailingRunStart(messages, (m) => m.role === "user"); const parts: string[] = []; for (let i = start; i < messages.length; i++) { const m = messages[i]; if (m === undefined) continue; newContentBytes += m.textBytes; newestRunImages += m.images; parts.push(m.text); } newestUserText = parts.join("\n"); } // Proportional share of the caller's prompt estimate, so this inherits // whatever tokenizer calibration the estimate already applied. const newContentTokens = req.promptBytes > 0 && newContentBytes > 0 ? Math.max(1, Math.round(promptTokens * (newContentBytes / req.promptBytes))) : 0; // Depth of the current agent loop: trailing tool results plus the assistant // tool-call turns interleaved with them. let toolLoopDepth = 0; for (let i = messages.length - 1; i >= 0; i--) { const m = messages[i]; if (m === undefined) break; if (m.role === "tool" || (m.role === "assistant" && m.toolCalls.length > 0)) toolLoopDepth++; else break; } let turnDepth = 0; let toolSchemaBytes = 0; const toolNames = new Set(); for (const t of req.tools) toolSchemaBytes += t.schemaBytes; for (const m of messages) { if (m.role === "user" || m.role === "assistant") turnDepth++; if (m.role === "assistant") for (const tc of m.toolCalls) toolNames.add(tc.name); if (m.toolName !== undefined) toolNames.add(m.toolName); } // The last few assistant tool calls, most-recent first, for stuck-loop // detection. A byte-identical call re-issued within this window means the // agent is going in circles even when the repeat is not adjacent — the case // the strict "last two identical" check misses. const RECENT_CALLS = 4; const recentCalls: { name: string; args: string }[] = []; scanCalls: for (let i = messages.length - 1; i >= 0; i--) { const m = messages[i]; if (m === undefined || m.role !== "assistant") continue; for (let j = m.toolCalls.length - 1; j >= 0; j--) { const tc = m.toolCalls[j]; if (tc === undefined) continue; recentCalls.push({ name: tc.name, args: tc.argsJson }); if (recentCalls.length >= RECENT_CALLS) break scanCalls; } } const repeatedToolCall = recentCalls.length >= 2 && recentCalls[0]?.name === recentCalls[1]?.name && recentCalls[0]?.args === recentCalls[1]?.args; let circularToolCall = false; for (let a = 0; a < recentCalls.length && !circularToolCall; a++) { for (let b = a + 1; b < recentCalls.length; b++) { if (recentCalls[a]?.name === recentCalls[b]?.name && recentCalls[a]?.args === recentCalls[b]?.args) { circularToolCall = true; break; } } } // The tool run to judge for failure: the trailing run on a continuation, or // the run that sits immediately behind the newest user turn — the failure // the human just saw and is now responding to. The classifier weights the // two differently (a mechanical retry is damped, a user-visible failure is // not), so the second case has to be detectable here or that branch is dead. let lastToolFailed = false; let failureScanFrom = -1; if (isToolResultContinuation) { failureScanFrom = messages.length - 1; } else if (tail?.role === "user") { const start = trailingRunStart(messages, (m) => m.role === "user"); if (messages[start - 1]?.role === "tool") failureScanFrom = start - 1; } if (failureScanFrom >= 0) { scanResults: for (let i = failureScanFrom; i >= 0; i--) { const m = messages[i]; if (m === undefined || m.role !== "tool") break; for (const re of TOOL_FAILURE_MARKERS) { if (re.test(m.text)) { lastToolFailed = true; break scanResults; } } } } // Fenced code blocks: odd segments of a ``` split. Byte count includes the // language tag line — close enough for a signal, and allocation-free per block. let codeBlocks = 0; let codeBytes = 0; if (newestUserText.includes("```")) { const parts = newestUserText.split("```"); codeBlocks = (parts.length - 1) >> 1; for (let i = 1; i + 1 < parts.length; i += 2) codeBytes += Buffer.byteLength(parts[i] ?? ""); } let questionCount = 0; for (let i = 0; i < newestUserText.length; i++) { if (newestUserText.charCodeAt(i) === 63) questionCount++; } const complexityKeywords: string[] = []; for (const [id, re] of COMPLEXITY_KEYWORDS) if (re.test(newestUserText)) complexityKeywords.push(id); const trivialityKeywords: string[] = []; for (const [id, re] of TRIVIALITY_KEYWORDS) if (re.test(newestUserText)) trivialityKeywords.push(id); const trimmed = newestUserText.trim(); const terminators = trimmed.match(/[.!?]+(?:\s|$)/g); const isTerseInstruction = trimmed.length > 0 && Buffer.byteLength(trimmed) <= TERSE_MAX_BYTES && codeBlocks === 0 && (terminators === null ? 0 : terminators.length) <= 1; // Prompt anatomy: where the bytes sit. Cheap (one pass over textBytes) and // content-free, so it is safe to record on every row. const anatomy: PromptAnatomy = { messages: 0, systemBytes: 0, userBytes: 0, assistantBytes: 0, toolBytes: 0, olderHalfBytes: 0, staleToolBytes: 0 }; const nonSystem = messages.filter((m) => m.role !== "system"); const olderHalfEnd = Math.floor(nonSystem.length / 2); const staleEnd = Math.max(0, nonSystem.length - 20); nonSystem.forEach((m, i) => { if (i < olderHalfEnd) anatomy.olderHalfBytes += m.textBytes; if (m.role === "tool" && i < staleEnd) anatomy.staleToolBytes += m.textBytes; }); for (const m of messages) { anatomy.messages++; if (m.role === "system") anatomy.systemBytes += m.textBytes; else if (m.role === "user") anatomy.userBytes += m.textBytes; else if (m.role === "assistant") anatomy.assistantBytes += m.textBytes; else anatomy.toolBytes += m.textBytes; } // Read-only tool loop: the assistant call behind a tool-result tail used // only tools that look at things. Tool names are the harness's own; the // set covers omp's built-ins and their common aliases. let readOnlyToolTail = false; if (isToolResultContinuation) { for (let i = messages.length - 1; i >= 0; i--) { const m = messages[i]; if (m === undefined || m.role !== "assistant") continue; if (m.toolCalls.length > 0) readOnlyToolTail = m.toolCalls.every((tc) => READ_ONLY_TOOLS.has(tc.name.toLowerCase())); break; } } return { promptTokens, newContentTokens, turnDepth, toolCount: req.tools.length, toolSchemaBytes, isToolResultContinuation, toolLoopDepth, distinctToolsUsed: toolNames.size, lastToolFailed, repeatedToolCall, circularToolCall, hasImages: req.hasImages, hasNewImage: newestRunImages > 0, codeBlocks, codeBytes, looksLikeDiff: DIFF_RE.test(newestUserText), complexityKeywords, trivialityKeywords, requestedReasoning: req.reasoning, questionCount, isTerseInstruction, anatomy, isSubagent: req.isSubagent, readOnlyToolTail, }; } /** Tools that read state without changing it, in omp, Claude Code and Hermes naming. */ export const READ_ONLY_TOOLS: ReadonlySet = new Set([ "read", "read_file", "grep", "glob", "ls", "list", "list_dir", "find", "lsp", "ast_grep", "search", "web_search", "web_fetch", "webfetch", "websearch", "fetch", "cat", "view", "inspect_image", "todo", ]);