{"version":3,"file":"token-accounting.d.ts","sourceRoot":"","sources":["../../../src/core/context-runtime/token-accounting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,WAAW,GAAG,YAAY,GAAG,uBAAuB,CAAC;AAEjG;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,GAAG,eAAe,GAAG,KAAK,CAAC;AAE/G,MAAM,WAAW,iBAAiB;IACjC,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,YAAY,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,gBAAgB,EAAE,MAAM,CAAC;IACzB,0EAAwE;IACxE,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAeD,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAS7C;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,CAyB3D;AAQD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CACjC,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,YAAsB,EACpC,OAAO,GAAE,sBAA2B,GAClC,MAAM,CAeR;AAED,gFAAgF;AAChF,wBAAgB,qBAAqB,CAAC,oBAAoB,EAAE,MAAM,GAAG,SAAS,GAAG,mBAAmB,CAGnG","sourcesContent":["/**\n * Tiered token accounting for context virtualization (2.6.0 hardening).\n *\n * The governor must never under-count against a model's REAL context limit.\n * Before this module, the sole safety basis was `chars / 4`, which is a good\n * approximation for English prose but dangerously under-counts content that a\n * real tokenizer expands:\n *\n *   - CJK / non-ASCII text (≈1 token per code point, not 1 per 4 chars)\n *   - emoji / astral-plane code points (≥1 token each, often more)\n *   - minified JSON / dense punctuation (≈1 token per symbol)\n *   - code (symbols and identifiers tokenize finer than prose)\n *\n * Tiering (best to worst):\n *\n *   exact                 -> provider/model tokenizer when available (not yet wired)\n *   tokenizer             -> local deterministic tokenizer (not yet wired)\n *   calibrated            -> bounded uplift derived from provider-reported usage\n *   conservative-estimate -> content-class aware fallback (always biased safe)\n *\n * No exact tokenizer dependency is added: the fallback is deterministic,\n * content-class aware, and biased toward over-estimation.\n */\n\nexport type TokenAccountingMode = \"exact\" | \"tokenizer\" | \"calibrated\" | \"conservative-estimate\";\n\n/**\n * Content class used to pick the conservative divisor. Lower divisor = more\n * conservative (more tokens per character).\n */\nexport type ContentClass = \"prose\" | \"code\" | \"json\" | \"tool-schema\" | \"tool-result\" | \"system-prompt\" | \"log\";\n\nexport interface TextTokenAnalysis {\n\t/** ASCII characters (code points < 0x80). */\n\tasciiChars: number;\n\t/** ASCII punctuation/symbol characters (not alphanumeric, not whitespace). */\n\tasciiSymbols: number;\n\t/** Non-ASCII BMP code points (CJK, accented Latin, etc.). */\n\tnonAsciiBmpChars: number;\n\t/** Astral-plane code points (surrogate pairs — emoji, rare scripts). */\n\tastralChars: number;\n\t/** Ratio of ASCII symbols to total ASCII (0..1). */\n\tsymbolRatio: number;\n}\n\nexport interface TokenAccountingOptions {\n\t/**\n\t * Bounded conservative uplift multiplier from provider-usage calibration.\n\t * Must be >= 1; only ever makes estimates larger, never smaller.\n\t */\n\tcalibratedMultiplier?: number;\n}\n\nconst DIVISORS: Record<ContentClass, number> = {\n\tprose: 4,\n\tcode: 3,\n\tjson: 2.5,\n\t\"tool-schema\": 3,\n\t\"tool-result\": 3,\n\t\"system-prompt\": 4,\n\tlog: 3,\n};\n\n/** Above this ASCII symbol density the estimator switches to symbol 1:1 accounting. */\nconst DENSE_SYMBOL_RATIO = 0.15;\n\n/** Bound the calibrated uplift so one outlier cannot cripple the budget. */\nexport const MAX_CALIBRATED_MULTIPLIER = 1.5;\n\nfunction isAsciiSymbol(ch: string): boolean {\n\tconst cp = ch.codePointAt(0)!;\n\tif (cp < 0x21 || cp > 0x7e) return false;\n\tif ((cp >= 0x30 && cp <= 0x39) || (cp >= 0x41 && cp <= 0x5a) || (cp >= 0x61 && cp <= 0x7a)) return false;\n\treturn true;\n}\n\n/**\n * Analyze a string into the token-relevant character classes. Iterates by code\n * point (not UTF-16 code unit) so surrogate pairs are counted as one astral char.\n */\nexport function analyzeText(text: string): TextTokenAnalysis {\n\tlet asciiChars = 0;\n\tlet asciiSymbols = 0;\n\tlet nonAsciiBmpChars = 0;\n\tlet astralChars = 0;\n\n\tfor (const ch of text) {\n\t\tconst cp = ch.codePointAt(0)!;\n\t\tif (cp < 0x80) {\n\t\t\tasciiChars += 1;\n\t\t\tif (isAsciiSymbol(ch)) asciiSymbols += 1;\n\t\t} else if (cp <= 0xffff) {\n\t\t\tnonAsciiBmpChars += 1;\n\t\t} else {\n\t\t\tastralChars += 1;\n\t\t}\n\t}\n\n\treturn {\n\t\tasciiChars,\n\t\tasciiSymbols,\n\t\tnonAsciiBmpChars,\n\t\tastralChars,\n\t\tsymbolRatio: asciiChars > 0 ? asciiSymbols / asciiChars : 0,\n\t};\n}\n\nfunction applyCalibratedUplift(tokens: number, multiplier: number | undefined): number {\n\tif (multiplier === undefined || multiplier <= 1) return tokens;\n\tconst bounded = Math.min(Math.max(multiplier, 1), MAX_CALIBRATED_MULTIPLIER);\n\treturn Math.ceil(tokens * bounded);\n}\n\n/**\n * Conservative token estimate for a single text payload.\n *\n * Guarantees (bias):\n *   - ASCII prose: ~1 token / 4 chars (matches the historical heuristic).\n *   - Code / JSON / logs: ~1 token / 3 chars or finer.\n *   - Dense punctuation (minified JSON): symbols counted 1:1, remaining /4.\n *   - Non-ASCII BMP (CJK): 1 token per code point.\n *   - Astral (emoji): 2 tokens per code point.\n *\n * These are deliberately upper-biased for the content classes a coding agent\n * actually encounters.\n */\nexport function estimateTextTokens(\n\ttext: string,\n\tcontentClass: ContentClass = \"prose\",\n\toptions: TokenAccountingOptions = {},\n): number {\n\tif (!text) return 0;\n\tconst a = analyzeText(text);\n\tconst asciiNonSymbols = Math.max(0, a.asciiChars - a.asciiSymbols);\n\n\tlet tokens: number;\n\tif (a.symbolRatio >= DENSE_SYMBOL_RATIO) {\n\t\t// Dense punctuation: every symbol is roughly its own token.\n\t\ttokens = Math.ceil(asciiNonSymbols / 4) + a.asciiSymbols + a.nonAsciiBmpChars + a.astralChars * 2;\n\t} else {\n\t\tconst divisor = DIVISORS[contentClass] ?? DIVISORS.prose;\n\t\ttokens = Math.ceil(a.asciiChars / divisor) + a.nonAsciiBmpChars + a.astralChars * 2;\n\t}\n\n\treturn applyCalibratedUplift(tokens, options.calibratedMultiplier);\n}\n\n/** Resolve the effective accounting mode from the current calibration state. */\nexport function resolveAccountingMode(calibratedMultiplier: number | undefined): TokenAccountingMode {\n\tif (calibratedMultiplier !== undefined && calibratedMultiplier > 1) return \"calibrated\";\n\treturn \"conservative-estimate\";\n}\n"]}