{"version":3,"file":"tokenization_utils.d.ts","sourceRoot":"","sources":["../src/tokenization_utils.js"],"names":[],"mappings":"AAiBA;;GAEG;AAEH;;;;;GAKG;AACH,6DAJW,MAAM,WACN,0BAA0B,GACxB,OAAO,CAAC,GAAG,EAAE,CAAC,CAO1B;AAED;;;;GAIG;AACH,+CAHW,MAAM,GACJ,MAAM,EAAE,CAiBpB;AAkvBD;;;;;;;GAOG;AACH,gDANW,mBAAmB,cACnB,MAAM,GAAC,MAAM,EAAE,qDAmDzB;0FAvnB8E,2BAA2B;AA9D1G;;;GAGG;AAEH;;;;GAIG;AAEH;;;;;;GAMG;AAEH;;;;;;;;;;;GAWG;AAEH;;GAEG;AAEH;;;;;;;;;;;;;;;;GAgBG;AAEH;;;;;GAKG;AAEH;IA+DI;;;;;;;;OAQG;IACH,sDANW,MAAM,yEACN,0BAA0B,GAGxB,OAAO,CAAC,mBAAmB,CAAC,CAgBxC;IAjFD,+BAA8B;IAE9B,qBAAuB;IASnB,oBAAmC;IACnC,sBAAuC;IACvC,sBAA+D;IAE/D,YAA6B;IAK7B,mBAAmE;IACnE,sBAAiE;IAEjE,kBAA8E;IAC9E,qBAA+D;IAE/D,kBAAiE;IACjE,qBAA+D;IAE/D,kBAAiE;IACjE,qBAA+D;IAE/D,kBAAiE;IACjE,qBAA+D;IAE/D,kBAAiE;IACjE,qBAA+D;IAE/D,mBAA0D;IAa1D,wCAAyC;IAGzC,0BAAsD;IACtD,6BAA8D;IA4BlE,iCAEC;IAED,4BAEC;IAED,yBAEC;IACD,yBAEC;IAED;;;;;;OAMG;IACH,sBAJ+B,CAAC,SAAlB,MAAM,GAAC,MAAM,EAAG,UACnB,CAAC,GACC,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAQhD;IAED;;;;;;;OAOG;IACH,MAN+B,KAAK,SAAtB,MAAM,GAAC,MAAM,EAAG,EACN,aAAa,SAAxB,OAAS,eACX,KAAK,YACL,oBAAoB,CAAC,KAAK,EAAE,aAAa,CAAC,GACxC,aAAa,CAAC,iBAAiB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAuJlE;IAED;;;;;OAKG;IACH,mBAHW,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,GAAC,IAAI,CAKzB;IAED;;;;;;;;;;OAUG;IACH,qBAWC;IAED;;;;;;;OAOG;IACH,eANW,MAAM,iCAEd;QAA8B,IAAI,GAA1B,MAAM,GAAC,IAAI;QACO,kBAAkB,GAApC,OAAO;KACf,GAAU,MAAM,EAAE,CAIpB;IAED;;;;;;;;;OASG;IACH,aAPW,MAAM,6DAEd;QAA8B,SAAS,GAA/B,MAAM,GAAC,IAAI;QACO,kBAAkB,GAApC,OAAO;QACgB,qBAAqB,GAA5C,OAAO,GAAC,IAAI;KACpB,GAAU,MAAM,EAAE,CAQpB;IAED;;;;;OAKG;IACH,oBAJW,MAAM,EAAE,EAAE,GAAC,MAAM,sBAEf,MAAM,EAAE,CAOpB;IAED;;;;;;;;;;OAUG;IACH,kBARW,MAAM,EAAE,GAAC,MAAM,EAAE,GAAC,MAAM,gBAEhC;QAA8B,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;KAEf,GAAU,MAAM,CAalB;IAED;;;;;;;;OAQG;IACH,yBAPW,MAAM,EAAE,GAAC,MAAM,EAAE,yDAEzB;QAA8B,mBAAmB,GAAzC,OAAO;QACoB,4BAA4B,GAAvD,OAAO,GAAC,IAAI;KAEpB,GAAU,MAAM,CAOlB;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,6CAZG;QAA8B,aAAa,GAAnC,MAAM,GAAC,IAAI;QAIQ,KAAK,GAAxB,KAAQ;KAMhB,GAAU,MAAM,CAsClB;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4DG;IACH,oBAhCwB,SAAS,SAApB,OAAS,SACE,aAAa,SAAxB,OAAS,SACE,WAAW,SAAtB,OAAS,uBAJX,OAAO,EAAE,YAMjB;QAA8B,aAAa,GAAnC,MAAM,GAAC,IAAI;QAEQ,KAAK,GAAxB,KAAQ;QAM2B,SAAS,GAA5C,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;QAMN,qBAAqB,GAAvC,OAAO;QAIa,QAAQ,GAA5B,SAAS;QACS,OAAO,GAAzB,OAAO;QACW,UAAU,GAA5B,OAAO;QACe,UAAU,GAAhC,MAAM,GAAC,IAAI;QAEa,aAAa,GAArC,aAAa;QACS,WAAW,GAAjC,WAAW;QACM,gBAAgB;KACzC,GAAU,uBAAuB,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAkE1E;CACJ;yCApxBY,OAAO,gBAAgB,EAAE,iBAAiB;0BAkD1C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAE;2BAMlD;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,kBAAkB,EAAE,QAAQ,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAE;;;;;6BAQ3F,WAAW,GAAG,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,GAAG,EAAE,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;CAAE;;;;;UAOrE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;;;;aAC/C,MAAM,GAAG,cAAc,EAAE;;mCAgFR,KAAK,SAAtB,MAAM,GAAC,MAAM,EAAG,IACjB,KAAK,SAAS,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE;8BAI1B,KAAK,SAAtB,MAAM,GAAC,MAAM,EAAG,EACN,aAAa,SAAxB,OAAS,WACT,aAAa,SAAS,IAAI,GAAG,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC;0BAInE,KAAK;;;;eAEJ,KAAK;;;;oBACL,KAAK;;;;qBACL,KAAK;;iCAIY,KAAK,SAAtB,MAAM,GAAC,MAAM,EAAG,EACN,aAAa,SAAxB,OAAS;;;;gBAER,KAAK,SAAS,MAAM,GAAG,MAAM,GAAC,IAAI,GAAG,MAAM,EAAE,GAAC,IAAI;;;;cAClD,OAAO,GAAC,YAAY;;;;yBACpB,OAAO;;;;iBACP,OAAO,GAAC,IAAI;;;;iBACZ,MAAM,GAAC,IAAI;;;;oBACX,aAAa;;;;4BACb,OAAO,GAAC,IAAI;;0CAIb,CAAC,KAAK,SAAS,MAAM,GAAG,MAAM,EAAE,EAAE,aAAa,SAAS,OAAO,GAAG,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,oBAAoB,CAAC,KAAK,EAAE,aAAa,CAAC,KAAK,aAAa,CAAC,iBAAiB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;qCAIzL,SAAS,SAApB,OAAS,SACE,aAAa,SAAxB,OAAS,SACE,WAAW,SAAtB,OAAS;;;;oBAER,MAAM,GAAC,IAAI;;;;YACX,KAAQ,GAAC,IAAI;;;;gBACb,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAC,IAAI;;;;4BAC7B,OAAO;;;;eACP,SAAS;;;;cACT,OAAO;;;;iBACP,OAAO;;;;iBACP,MAAM,GAAC,IAAI;;;;oBACX,aAAa;;;;kBACb,WAAW;;;;;;oCAKD,SAAS,SAApB,OAAS,SACE,aAAa,SAAxB,OAAS,SACE,WAAW,SAAtB,OAAS,WACT,SAAS,SAAS,KAAK,GAAG,MAAM,GAAG,WAAW,SAAS,KAAK,GAAG,iBAAiB,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,aAAa,CAAC,iBAAiB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;uBAtNvJ,mBAAmB;0BAPhB,yBAAyB"}