/** * Tiered token accounting for context virtualization (2.6.0 hardening). * * The governor must never under-count against a model's REAL context limit. * Before this module, the sole safety basis was `chars / 4`, which is a good * approximation for English prose but dangerously under-counts content that a * real tokenizer expands: * * - CJK / non-ASCII text (≈1 token per code point, not 1 per 4 chars) * - emoji / astral-plane code points (≥1 token each, often more) * - minified JSON / dense punctuation (≈1 token per symbol) * - code (symbols and identifiers tokenize finer than prose) * * Tiering (best to worst): * * exact -> provider/model tokenizer when available (not yet wired) * tokenizer -> local deterministic tokenizer (not yet wired) * calibrated -> bounded uplift derived from provider-reported usage * conservative-estimate -> content-class aware fallback (always biased safe) * * No exact tokenizer dependency is added: the fallback is deterministic, * content-class aware, and biased toward over-estimation. */ export type TokenAccountingMode = "exact" | "tokenizer" | "calibrated" | "conservative-estimate"; /** * Content class used to pick the conservative divisor. Lower divisor = more * conservative (more tokens per character). */ export type ContentClass = "prose" | "code" | "json" | "tool-schema" | "tool-result" | "system-prompt" | "log"; export interface TextTokenAnalysis { /** ASCII characters (code points < 0x80). */ asciiChars: number; /** ASCII punctuation/symbol characters (not alphanumeric, not whitespace). */ asciiSymbols: number; /** Non-ASCII BMP code points (CJK, accented Latin, etc.). */ nonAsciiBmpChars: number; /** Astral-plane code points (surrogate pairs — emoji, rare scripts). */ astralChars: number; /** Ratio of ASCII symbols to total ASCII (0..1). */ symbolRatio: number; } export interface TokenAccountingOptions { /** * Bounded conservative uplift multiplier from provider-usage calibration. * Must be >= 1; only ever makes estimates larger, never smaller. */ calibratedMultiplier?: number; } /** Bound the calibrated uplift so one outlier cannot cripple the budget. */ export declare const MAX_CALIBRATED_MULTIPLIER = 1.5; /** * Analyze a string into the token-relevant character classes. Iterates by code * point (not UTF-16 code unit) so surrogate pairs are counted as one astral char. */ export declare function analyzeText(text: string): TextTokenAnalysis; /** * Conservative token estimate for a single text payload. * * Guarantees (bias): * - ASCII prose: ~1 token / 4 chars (matches the historical heuristic). * - Code / JSON / logs: ~1 token / 3 chars or finer. * - Dense punctuation (minified JSON): symbols counted 1:1, remaining /4. * - Non-ASCII BMP (CJK): 1 token per code point. * - Astral (emoji): 2 tokens per code point. * * These are deliberately upper-biased for the content classes a coding agent * actually encounters. */ export declare function estimateTextTokens(text: string, contentClass?: ContentClass, options?: TokenAccountingOptions): number; /** Resolve the effective accounting mode from the current calibration state. */ export declare function resolveAccountingMode(calibratedMultiplier: number | undefined): TokenAccountingMode; //# sourceMappingURL=token-accounting.d.ts.map