{"version":3,"file":"tokens.cjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport type EncodingName = 'o200k_base' | 'claude';\n\nconst tokenizers: Partial> = {};\n\nasync function getTokenizer(\n encoding: EncodingName = 'o200k_base'\n): Promise {\n const cached = tokenizers[encoding];\n if (cached) {\n return cached;\n }\n const data =\n encoding === 'claude'\n ? await import('ai-tokenizer/encoding/claude')\n : await import('ai-tokenizer/encoding/o200k_base');\n const instance = new Tokenizer(data);\n tokenizers[encoding] = instance;\n return instance;\n}\n\nexport function encodingForModel(model: string): EncodingName {\n if (model.toLowerCase().includes('claude')) {\n return 'claude';\n }\n return 'o200k_base';\n}\n\nexport function getTokenCountForMessage(\n message: BaseMessage,\n getTokenCount: (text: string) => number\n): number {\n const tokensPerMessage = 3;\n\n const processValue = (value: unknown): void => {\n if (Array.isArray(value)) {\n for (const item of value) {\n if (\n !item ||\n !item.type ||\n item.type === ContentTypes.ERROR ||\n item.type === ContentTypes.IMAGE_URL\n ) {\n continue;\n }\n\n if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n const toolName = item.tool_call?.name || '';\n if (toolName != null && toolName && typeof toolName === 'string') {\n numTokens += getTokenCount(toolName);\n }\n\n const args = item.tool_call?.args || '';\n if (args != null && args && typeof args === 'string') {\n numTokens += getTokenCount(args);\n }\n\n const output = item.tool_call?.output || '';\n if (output != null && output && typeof output === 'string') {\n numTokens += getTokenCount(output);\n }\n continue;\n }\n\n const nestedValue = item[item.type];\n\n if (!nestedValue) {\n continue;\n }\n\n processValue(nestedValue);\n }\n } else if (typeof value === 'string') {\n numTokens += getTokenCount(value);\n } else if (typeof value === 'number') {\n numTokens += getTokenCount(value.toString());\n } else if (typeof value === 'boolean') {\n numTokens += getTokenCount(value.toString());\n }\n };\n\n let numTokens = tokensPerMessage;\n processValue(message.content);\n return numTokens;\n}\n\n/**\n * Creates a token counter function using the specified encoding.\n * Lazily loads the encoding data on first use via dynamic import.\n */\nexport const createTokenCounter = async (\n encoding: EncodingName = 'o200k_base'\n): Promise<(message: BaseMessage) => number> => {\n const tok = await getTokenizer(encoding);\n const countTokens = (text: string): number => tok.count(text);\n return (message: BaseMessage): number =>\n getTokenCountForMessage(message, countTokens);\n};\n\n/** Utility to manage the token encoder lifecycle explicitly. */\nexport const TokenEncoderManager = {\n async initialize(): Promise {\n // No-op: ai-tokenizer is synchronously initialized from bundled data.\n },\n\n reset(): void {\n for (const key of Object.keys(tokenizers)) {\n delete tokenizers[key as EncodingName];\n }\n },\n\n isInitialized(): boolean {\n return Object.keys(tokenizers).length > 0;\n },\n};\n"],"names":["Tokenizer","ContentTypes"],"mappings":";;;;;AAMA,MAAM,UAAU,GAA6C,EAAE;AAE/D,eAAe,YAAY,CACzB,QAAA,GAAyB,YAAY,EAAA;AAErC,IAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;IACnC,IAAI,MAAM,EAAE;AACV,QAAA,OAAO,MAAM;IACf;AACA,IAAA,MAAM,IAAI,GACR,QAAQ,KAAK;AACX,UAAE,MAAM,OAAO,8BAA8B;AAC7C,UAAE,MAAM,OAAO,kCAAkC,CAAC;AACtD,IAAA,MAAM,QAAQ,GAAG,IAAIA,qBAAS,CAAC,IAAI,CAAC;AACpC,IAAA,UAAU,CAAC,QAAQ,CAAC,GAAG,QAAQ;AAC/B,IAAA,OAAO,QAAQ;AACjB;AAEM,SAAU,gBAAgB,CAAC,KAAa,EAAA;IAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;AAC1C,QAAA,OAAO,QAAQ;IACjB;AACA,IAAA,OAAO,YAAY;AACrB;AAEM,SAAU,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAKC,kBAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,EACpC;oBACA;gBACF;AAEA,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;oBACtC;oBAEA,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;oBAClC;oBAEA,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;oBACpC;oBACA;gBACF;gBAEA,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;gBACF;gBAEA,YAAY,CAAC,WAAW,CAAC;YAC3B;QACF;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;QACnC;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AACF,IAAA,CAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA;;;AAGG;MACU,kBAAkB,GAAG,OAChC,QAAA,GAAyB,YAAY,KACQ;AAC7C,IAAA,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC;AACxC,IAAA,MAAM,WAAW,GAAG,CAAC,IAAY,KAAa,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;IAC7D,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;AACjD;AAEA;AACO,MAAM,mBAAmB,GAAG;AACjC,IAAA,MAAM,UAAU,GAAA;;IAEhB,CAAC;IAED,KAAK,GAAA;QACH,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;AACzC,YAAA,OAAO,UAAU,CAAC,GAAmB,CAAC;QACxC;IACF,CAAC;IAED,aAAa,GAAA;QACX,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC;IAC3C,CAAC;;;;;;;;"}