{"version":3,"file":"relevance.mjs","names":[],"sources":["../../../../src/batteries/context/thrift/relevance.ts"],"sourcesContent":["/**\n * Relevance-based history-turn selection — the companion to {@link subtractToFit} that decides WHICH\n * prior turns are worth replaying at all, before the subtractive pass ever runs.\n *\n * @module @nhtio/adk/batteries/context/thrift/relevance\n *\n * @remarks\n * {@link @nhtio/adk/batteries/context/thrift/subtractive_pass!subtractToFit} sheds a working set's\n * OLDEST turns first once the budget is blown — a purely recency-based (FIFO) policy. This module is\n * a smarter alternative for the turn-selection step upstream of that: instead of \"the last N turns,\"\n * walk the ENTIRE history (no turn-count cap) and keep any turn whose content is LEXICALLY RELEVANT to\n * the current query, regardless of how old it is, while always keeping the most recent `keepRecent`\n * turns verbatim (coreference — \"it\", \"that file\" — needs the immediately preceding turns present no\n * matter what).\n *\n * This is a genuine head-to-head-evaluated alternative to naive recency, not a hypothetical: in the\n * flagship reference agent this battery was extracted from, {@link selectRelevantTurns} was the\n * TREATMENT arm and {@link selectNaiveTurns} was the BASELINE arm of the same evaluation the battery\n * barrel documents. Use `selectRelevantTurns` when you want the evaluated win; `selectNaiveTurns`\n * remains exported as an honest, drop-in comparison baseline (or for a caller who simply wants FIFO).\n *\n * Zero imports, same as every module in this battery — token measurement is via an injected\n * {@link EstimateTokensFn}, never a bundled tokenizer.\n */\n\nimport type { EstimateTokensFn } from './contracts'\n\nexport type { EstimateTokensFn }\n\n/**\n * The relevance floor's lower bound — the minimum fraction of the current query's content words a\n * turn must share to be kept, used when the window is nearly empty (permissive: keep almost anything\n * that shares ANY real overlap).\n *\n * @remarks\n * CALIBRATED against a relevance oracle over a 94-turn stress corpus, triple-confirmed across three\n * model families acting as independent judges (gpt-5.5, gemini-3.5-flash, claude-haiku-4.5): the\n * F1-optimal floor (best balance of precision and recall when window budget is abundant) landed at\n * `0.125` for gpt-5.5 and claude-haiku-4.5 exactly, and `0.111` for gemini-3.5-flash — `0.125` was\n * chosen as the shared default (a hair stricter than gemini's number, i.e. slightly more permissive\n * than not, which the calibration run characterized as \"slightly permissive\" rather than\n * under-inclusive — the safer direction to err when the window has room to spare).\n */\nexport const RELEVANCE_FLOOR_MIN = 0.125\n\n/**\n * The relevance floor's upper bound — the minimum shared-content-word fraction required when the\n * window is nearly full (strict: only keep turns with strong, unambiguous overlap).\n *\n * @remarks\n * CALIBRATED against the same triple-oracle stress corpus as {@link RELEVANCE_FLOOR_MIN}: the\n * high-precision floor (precision ≥ 0.9, i.e. \"when in doubt about whether this survives, don't miss\n * on the side of dropping something the user is about to ask about\") landed BIT-IDENTICAL at\n * `0.4286` across all three judge models (gpt-5.5, gemini-3.5-flash, claude-haiku-4.5) — a striking\n * cross-model agreement that grounds this as a real property of the corpus, not judge idiosyncrasy.\n */\nexport const RELEVANCE_FLOOR_MAX = 0.43\n\n/**\n * The exponent shaping how the relevance floor scales between {@link RELEVANCE_FLOOR_MIN} and\n * {@link RELEVANCE_FLOOR_MAX} as window utilization rises from 0 to 1 (see\n * {@link scaledRelevanceFloor}).\n *\n * @remarks\n * `2` — a CONVEX curve (utilization raised to this power) — was chosen deliberately over a linear\n * scale: it stays permissive across most of the window's life (older turns keep surviving on weak\n * overlap while there's room to spare) and only tightens sharply once the window is GENUINELY filling\n * up, rather than progressively squeezing out marginal turns from the very first token of pressure.\n * The convex shape is itself part of what the calibration run validated, not an arbitrary choice.\n */\nexport const RELEVANCE_FLOOR_CURVE = 2\n\n/**\n * Scale the relevance floor between {@link RELEVANCE_FLOOR_MIN} (empty window, permissive) and\n * {@link RELEVANCE_FLOOR_MAX} (full window, strict) by a convex curve on window utilization — see\n * {@link RELEVANCE_FLOOR_CURVE} for why convex.\n *\n * @param utilization - How full the window already is, as a fraction in `[0, 1]` (values outside the\n *   range are clamped). Typically `olderTurnsTokens / historyBudget` — how much of the OLDER-turn\n *   budget is already spent by turns kept unconditionally (e.g. the `keepRecent` window).\n * @returns The minimum shared-content-word fraction (see {@link relevanceToQuery}) a turn must clear\n *   to survive selection at this utilization level.\n */\nexport function scaledRelevanceFloor(utilization: number): number {\n  const u = Math.min(1, Math.max(0, utilization))\n  return (\n    RELEVANCE_FLOOR_MIN +\n    (RELEVANCE_FLOOR_MAX - RELEVANCE_FLOOR_MIN) * Math.pow(u, RELEVANCE_FLOOR_CURVE)\n  )\n}\n\n/**\n * Extract the \"content word\" tokens of a string for lexical overlap scoring: lowercased alphanumeric\n * runs of length >= 4, deduplicated into a set. Deliberately crude (no stemming, no stopword list\n * beyond the length-4 floor) — this is a cheap, zero-model-call, zero-dependency relevance signal, not\n * a semantic one; the calibration in {@link RELEVANCE_FLOOR_MIN}/{@link RELEVANCE_FLOOR_MAX} was run\n * against exactly this scoring function, so changing it invalidates those constants.\n *\n * @param text - Text to extract content-word tokens from.\n * @returns The distinct content words found, lowercased.\n */\nexport function contentTokens(text: string): Set<string> {\n  const out = new Set<string>()\n  for (const m of text.toLowerCase().matchAll(/[a-z][a-z0-9]{3,}/g)) out.add(m[0])\n  return out\n}\n\n/**\n * Render a tool call's arguments into a short text fragment for inclusion in a turn's relevance text\n * (see {@link groupHistoryIntoTurns}) — best-effort `JSON.stringify`, truncated, empty on failure\n * (e.g. circular structures) or absent args.\n *\n * @param args - The tool call's arguments, in whatever shape the caller's tool-call record carries.\n * @returns A short (<=200 char) text fragment, or an empty string if `args` is nullish or\n *   unserializable.\n */\nexport function argText(args: unknown): string {\n  if (args === null || args === undefined) return ''\n  try {\n    return JSON.stringify(args).slice(0, 200)\n  } catch {\n    return ''\n  }\n}\n\n/**\n * What fraction of the QUERY's content words appear in `text` — the relevance of a prior turn to the\n * CURRENT query (not the reverse: a turn need not share all its own content, only enough of the\n * query's).\n *\n * @param text - The candidate turn's combined text (see `HistoryTurn.qa`).\n * @param queryTokens - The current query's content-word set, from {@link contentTokens}.\n * @returns A fraction in `[0, 1]`; `0` when `queryTokens` is empty (an empty query matches nothing).\n */\nexport function relevanceToQuery(text: string, queryTokens: ReadonlySet<string>): number {\n  if (queryTokens.size === 0) return 0\n  const tt = contentTokens(text)\n  let shared = 0\n  for (const q of queryTokens) if (tt.has(q)) shared++\n  return shared / queryTokens.size\n}\n\n/**\n * The minimal structural shape of a conversation message this module groups into turns. `createdAt`\n * is a lexicographically-sortable timestamp string (e.g. ISO-8601) — turns and tool calls are ordered\n * by string comparison (`localeCompare`), not parsed into a `Date`, so any consistently-formatted\n * sortable string works.\n *\n * @remarks\n * This is intentionally a DIFFERENT (simpler) shape than\n * {@link @nhtio/adk/batteries/context/thrift/contracts!WorkingMessage} — turn grouping runs UPSTREAM\n * of the subtractive pass, over raw conversation history, before it becomes working-set items. A\n * caller's real message type will usually carry more fields than this; because every function here\n * accepts a generic `M extends RelevanceMessage`, extra fields pass through untouched.\n */\nexport interface RelevanceMessage {\n  /** `'user'`, `'assistant'`, or any other role identifier — an `'assistant'` message closes the\n   *  current turn (see {@link groupHistoryIntoTurns}). */\n  role: string\n  /** The message's rendered text, folded into the turn's combined relevance text. */\n  content: string\n  /** Sortable creation timestamp (e.g. ISO-8601). */\n  createdAt: string\n}\n\n/**\n * The minimal structural shape of a tool call this module attaches to the turn it occurred in.\n */\nexport interface RelevanceToolCall {\n  /** Sortable creation timestamp (e.g. ISO-8601), compared against message timestamps to find which\n   *  turn a call belongs to. */\n  createdAt: string\n  /** The tool's name, folded into the turn's combined relevance text when present. Omit for a call\n   *  with no resolved tool name yet. */\n  tool?: string\n  /** The call's arguments, rendered via {@link argText} into the turn's combined relevance text. */\n  args?: unknown\n}\n\n/**\n * One grouped conversation turn: the messages from a user message through the next assistant message\n * (inclusive), plus any tool calls that occurred during it, and `qa` — the turn's full combined text\n * (every message's content plus every tool call's `name args` fragment) used for relevance scoring.\n */\nexport interface HistoryTurn<\n  M extends RelevanceMessage = RelevanceMessage,\n  TC extends RelevanceToolCall = RelevanceToolCall,\n> {\n  /** The turn's combined text (all message content + `tool argText(args)` fragments), the string\n   *  {@link relevanceToQuery} scores against. */\n  qa: string\n  /** The turn's timestamp, updated to the latest message folded into it — used for\n   *  `selectNaiveTurns`' recency ordering. */\n  createdAt: string\n  /** The messages belonging to this turn, in order. */\n  messages: M[]\n  /** The tool calls attributed to this turn (by timestamp — see {@link groupHistoryIntoTurns}). */\n  toolCalls: TC[]\n}\n\n/**\n * Group a flat message history (plus its tool calls) into {@link HistoryTurn}s: a new turn starts at\n * each message that begins a fresh exchange and closes at the next `'assistant'`-role message\n * (inclusive); every message up to the first turn boundary that has no preceding messages also starts\n * a turn. Tool calls are attributed to the LATEST turn whose first message precedes the call's\n * timestamp.\n *\n * @param messages - The full flat message history, in chronological order.\n * @param toolCalls - The full flat tool-call history, in any order (this function sorts by\n *   attribution, not the input order).\n * @returns The grouped turns, in chronological order.\n */\nexport function groupHistoryIntoTurns<\n  M extends RelevanceMessage = RelevanceMessage,\n  TC extends RelevanceToolCall = RelevanceToolCall,\n>(messages: readonly M[], toolCalls: readonly TC[]): Array<HistoryTurn<M, TC>> {\n  const turns: Array<HistoryTurn<M, TC>> = []\n  let cur: HistoryTurn<M, TC> | null = null\n  for (const m of messages) {\n    if (!cur) {\n      cur = { qa: m.content, createdAt: m.createdAt, messages: [m], toolCalls: [] }\n      turns.push(cur)\n      if (m.role === 'assistant') cur = null\n      continue\n    }\n    cur.messages.push(m)\n    cur.qa += `\\n${m.content}`\n    cur.createdAt = m.createdAt\n    if (m.role === 'assistant') cur = null\n  }\n  for (const tc of toolCalls) {\n    let target: HistoryTurn<M, TC> | null = null\n    for (const turn of turns) {\n      if (turn.messages[0].createdAt.localeCompare(tc.createdAt) <= 0) target = turn\n      else break\n    }\n    if (target) {\n      target.toolCalls.push(tc)\n      if (tc.tool) target.qa += `\\n${tc.tool} ${argText(tc.args)}`\n    }\n  }\n  return turns\n}\n\n/**\n * Options for {@link selectRelevantTurns}.\n */\nexport interface SelectRelevantTurnsOptions {\n  /**\n   * REQUIRED. Measures a turn's combined `qa` text's token cost. There is no default — this battery\n   * ships with no bundled tokenizer.\n   */\n  estimateTokens: EstimateTokensFn\n  /** The encoding to measure under. Default: `'cl100k_base'` (see\n   *  {@link @nhtio/adk/batteries/context/thrift/subtractive_pass!DEFAULT_ENCODING} for the rationale —\n   *  this module makes the identical choice for the identical reason). */\n  encoding?: string\n  /**\n   * How many of the MOST RECENT turns to always keep regardless of relevance score — coreference\n   * (\"it\", \"that file\", \"the one you just showed me\") needs the immediately preceding turns present no\n   * matter what a lexical-overlap score says about them. Default: `2`, the flagship reference agent's\n   * own calibrated value (`KEEP_RECENT_TURNS`).\n   */\n  keepRecent?: number\n  /**\n   * The token budget available for OLDER (non-`keepRecent`) history — used only to compute window\n   * UTILIZATION for {@link scaledRelevanceFloor} (how full is the older-history slice already, before\n   * any turns are dropped), not as a hard cap this function enforces itself (that is\n   * `subtractToFit`'s job, downstream). A `historyBudget` of `0` scores utilization as `0` (maximally\n   * permissive floor).\n   */\n  historyBudget: number\n  /** Override for {@link RELEVANCE_FLOOR_MIN}. Overriding without re-running the oracle calibration\n   *  forfeits the calibrated guarantee — see that constant's TSDoc before changing it. */\n  floorMin?: number\n  /** Override for {@link RELEVANCE_FLOOR_MAX}. Same caveat as `floorMin`. */\n  floorMax?: number\n  /** Override for {@link RELEVANCE_FLOOR_CURVE}. Same caveat as `floorMin`. */\n  floorCurve?: number\n}\n\n/**\n * Select which turns to replay by RELEVANCE to the current query: always keep the most recent\n * `keepRecent` turns, and among the OLDER turns keep any whose {@link relevanceToQuery} score against\n * the query's content words clears a floor that scales with how much of the older-history budget is\n * already spoken for (see {@link scaledRelevanceFloor}). Walks the ENTIRE history — there is no\n * turn-count cap — so a turn from far in the past survives if it is genuinely relevant, which a\n * recency-only policy (see {@link selectNaiveTurns}) can never do.\n *\n * @param turns - The full grouped history, chronological, from {@link groupHistoryIntoTurns}.\n * @param queryText - The current user query (or turn text) to score every older turn against.\n * @param options - See {@link SelectRelevantTurnsOptions}.\n * @returns The surviving turns, in their original chronological order.\n */\nexport function selectRelevantTurns<\n  M extends RelevanceMessage = RelevanceMessage,\n  TC extends RelevanceToolCall = RelevanceToolCall,\n>(\n  turns: ReadonlyArray<HistoryTurn<M, TC>>,\n  queryText: string,\n  options: SelectRelevantTurnsOptions\n): Array<HistoryTurn<M, TC>> {\n  const encoding = options.encoding ?? 'cl100k_base'\n  const keepRecent = options.keepRecent ?? 2\n  const floorMin = options.floorMin ?? RELEVANCE_FLOOR_MIN\n  const floorMax = options.floorMax ?? RELEVANCE_FLOOR_MAX\n  const floorCurve = options.floorCurve ?? RELEVANCE_FLOOR_CURVE\n  const floorFor = (utilization: number): number => {\n    const u = Math.min(1, Math.max(0, utilization))\n    return floorMin + (floorMax - floorMin) * Math.pow(u, floorCurve)\n  }\n  const queryTokens = contentTokens(queryText)\n  const recentStart = turns.length - keepRecent\n  let olderTokens = 0\n  for (let i = 0; i < recentStart; i++) olderTokens += options.estimateTokens(turns[i].qa, encoding)\n  const floor = floorFor(options.historyBudget > 0 ? olderTokens / options.historyBudget : 0)\n  return turns.filter((turn, idx) => {\n    if (idx >= recentStart) return true\n    return relevanceToQuery(turn.qa, queryTokens) >= floor\n  })\n}\n\n/**\n * Options for {@link selectNaiveTurns}.\n */\nexport interface SelectNaiveTurnsOptions {\n  /** REQUIRED. Measures a turn's combined `qa` text's token cost. */\n  estimateTokens: EstimateTokensFn\n  /** The encoding to measure under. Default: `'cl100k_base'`. */\n  encoding?: string\n}\n\n/**\n * The recency (FIFO) baseline: keep the newest turns, oldest-first-dropped, until `historyBudget` is\n * exhausted — walking backward from the newest turn and stopping (WITHOUT including) the first turn\n * that would push the running total over budget. The single newest turn is always kept even if it\n * alone exceeds `historyBudget`.\n *\n * @remarks\n * This function exists as an honest COMPARISON BASELINE, not a recommended default — it is the\n * \"before\" arm the flagship reference agent's evaluation measured {@link selectRelevantTurns} against.\n * The evaluation's own honest finding: naive recency is not merely worse in the general case, it can\n * COLLAPSE entirely on a reasoning-heavy model under enough context pressure — one matrix cell\n * observed the naive baseline degrade to keeping essentially nothing useful (effectively `floor 0.08`\n * / `3` turns of real signal) once the window filled, while relevance selection kept answering\n * correctly at the same pressure. Use `selectRelevantTurns` unless you specifically need a\n * recency-only policy or are reproducing that comparison yourself.\n *\n * @param turns - The full grouped history, chronological.\n * @param historyBudget - The token budget to fit turns into, newest-first.\n * @param options - See {@link SelectNaiveTurnsOptions}.\n * @returns The surviving turns, in their original chronological order.\n */\nexport function selectNaiveTurns<\n  M extends RelevanceMessage = RelevanceMessage,\n  TC extends RelevanceToolCall = RelevanceToolCall,\n>(\n  turns: ReadonlyArray<HistoryTurn<M, TC>>,\n  historyBudget: number,\n  options: SelectNaiveTurnsOptions\n): Array<HistoryTurn<M, TC>> {\n  const encoding = options.encoding ?? 'cl100k_base'\n  const kept: Array<HistoryTurn<M, TC>> = []\n  let used = 0\n  for (let i = turns.length - 1; i >= 0; i--) {\n    const cost = options.estimateTokens(turns[i].qa, encoding)\n    if (kept.length > 0 && used + cost > historyBudget) break\n    kept.push(turns[i])\n    used += cost\n  }\n  return kept.reverse()\n}\n"],"mappings":";;;;;;;;;;;;;;;AA2CA,IAAa,sBAAsB;;;;;;;;;;;;AAanC,IAAa,sBAAsB;;;;;;;;;;;;;AAcnC,IAAa,wBAAwB;;;;;;;;;;;;AAarC,SAAgB,qBAAqB,aAA6B;CAEhE,OACE,uBACC,sBAAsB,uBAAuB,KAAK,IAH3C,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,WAAW,CAGY,GAAA,CAAwB;AAEnF;;;;;;;;;;;AAYA,SAAgB,cAAc,MAA2B;CACvD,MAAM,sBAAM,IAAI,IAAY;CAC5B,KAAK,MAAM,KAAK,KAAK,YAAY,EAAE,SAAS,oBAAoB,GAAG,IAAI,IAAI,EAAE,EAAE;CAC/E,OAAO;AACT;;;;;;;;;;AAWA,SAAgB,QAAQ,MAAuB;CAC7C,IAAI,SAAS,QAAQ,SAAS,KAAA,GAAW,OAAO;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,IAAI,EAAE,MAAM,GAAG,GAAG;CAC1C,QAAQ;EACN,OAAO;CACT;AACF;;;;;;;;;;AAWA,SAAgB,iBAAiB,MAAc,aAA0C;CACvF,IAAI,YAAY,SAAS,GAAG,OAAO;CACnC,MAAM,KAAK,cAAc,IAAI;CAC7B,IAAI,SAAS;CACb,KAAK,MAAM,KAAK,aAAa,IAAI,GAAG,IAAI,CAAC,GAAG;CAC5C,OAAO,SAAS,YAAY;AAC9B;;;;;;;;;;;;;AAwEA,SAAgB,sBAGd,UAAwB,WAAqD;CAC7E,MAAM,QAAmC,CAAC;CAC1C,IAAI,MAAiC;CACrC,KAAK,MAAM,KAAK,UAAU;EACxB,IAAI,CAAC,KAAK;GACR,MAAM;IAAE,IAAI,EAAE;IAAS,WAAW,EAAE;IAAW,UAAU,CAAC,CAAC;IAAG,WAAW,CAAC;GAAE;GAC5E,MAAM,KAAK,GAAG;GACd,IAAI,EAAE,SAAS,aAAa,MAAM;GAClC;EACF;EACA,IAAI,SAAS,KAAK,CAAC;EACnB,IAAI,MAAM,KAAK,EAAE;EACjB,IAAI,YAAY,EAAE;EAClB,IAAI,EAAE,SAAS,aAAa,MAAM;CACpC;CACA,KAAK,MAAM,MAAM,WAAW;EAC1B,IAAI,SAAoC;EACxC,KAAK,MAAM,QAAQ,OACjB,IAAI,KAAK,SAAS,GAAG,UAAU,cAAc,GAAG,SAAS,KAAK,GAAG,SAAS;OACrE;EAEP,IAAI,QAAQ;GACV,OAAO,UAAU,KAAK,EAAE;GACxB,IAAI,GAAG,MAAM,OAAO,MAAM,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,IAAI;EAC3D;CACF;CACA,OAAO;AACT;;;;;;;;;;;;;;AAoDA,SAAgB,oBAId,OACA,WACA,SAC2B;CAC3B,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,aAAa,QAAQ,cAAc;CACzC,MAAM,WAAW,QAAQ,YAAA;CACzB,MAAM,WAAW,QAAQ,YAAA;CACzB,MAAM,aAAa,QAAQ,cAAA;CAC3B,MAAM,YAAY,gBAAgC;EAChD,MAAM,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,WAAW,CAAC;EAC9C,OAAO,YAAY,WAAW,YAAY,KAAK,IAAI,GAAG,UAAU;CAClE;CACA,MAAM,cAAc,cAAc,SAAS;CAC3C,MAAM,cAAc,MAAM,SAAS;CACnC,IAAI,cAAc;CAClB,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK,eAAe,QAAQ,eAAe,MAAM,GAAG,IAAI,QAAQ;CACjG,MAAM,QAAQ,SAAS,QAAQ,gBAAgB,IAAI,cAAc,QAAQ,gBAAgB,CAAC;CAC1F,OAAO,MAAM,QAAQ,MAAM,QAAQ;EACjC,IAAI,OAAO,aAAa,OAAO;EAC/B,OAAO,iBAAiB,KAAK,IAAI,WAAW,KAAK;CACnD,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAiCA,SAAgB,iBAId,OACA,eACA,SAC2B;CAC3B,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,OAAkC,CAAC;CACzC,IAAI,OAAO;CACX,KAAK,IAAI,IAAI,MAAM,SAAS,GAAG,KAAK,GAAG,KAAK;EAC1C,MAAM,OAAO,QAAQ,eAAe,MAAM,GAAG,IAAI,QAAQ;EACzD,IAAI,KAAK,SAAS,KAAK,OAAO,OAAO,eAAe;EACpD,KAAK,KAAK,MAAM,EAAE;EAClB,QAAQ;CACV;CACA,OAAO,KAAK,QAAQ;AACtB"}