{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.js"],"names":[],"mappings":";;;;AAyIA;;;;GAIG;AACH;IA0BI;;;;;;OAMG;IACH,gDAHa,cAAc,CAiB1B;IA9CD;;;OAGG;IACH,yBAmBC;IAjBG,YAAoB;IAEpB,uBAAuB;IACvB,OADW,MAAM,EAAE,CACJ;IAEf;;;OAGG;IACH,eAFU,IAAI,MAAM,EAAE,MAAM,CAAC,CAEC;IAE9B,kBAA6B;IAC7B,eAA0B;IAC1B,wBAAmC;IAEnC,uFAAuF;IACvF,UADW,OAAO,CACG;IA0BzB;;;;OAIG;IACH,cAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;IAED;;;;;OAKG;IACH,eAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;IAED;;;;OAIG;IACH,8BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAUpB;IAED;;;;OAIG;IACH,2BAHW,MAAM,EAAE,GACN,MAAM,EAAE,CAIpB;CACJ;;;;;AAgrDD;IAyGI;;;;;;;;OAQG;IACH,sDANW,MAAM,0EACN,iBAAiB,GAGf,QAAQ,mBAAmB,CAAC,CAoBxC;IAnID;;;;OAIG;IACH,sDAwEC;IApEG,uBAAiE;IACjE,4BAAyE;IASzE,sBAA4E;IAC5E,8BAA4E;IAG5E,iBAAwD;IAUxD,sBAAwB;IACxB,uBAAyB;IACzB,oBAAsB;IAmBtB,2BAEC;IAGD,mBAA8D;IAC9D,sBAAkE;IAElE,kBAAyE;IACzE,qBAAgE;IAEhE,kBAA4D;IAC5D,qBAAgE;IAEhE,sBAAwD;IAExD,6HAA6H;IAC7H,cADW,OAAO,CAC8B;IAEhD,kCAAwF;IAGxF,qBAA2B;IAG/B;;;;;OAKG;IACH,wCAJc,MAAM,KACP,MAAM,GAAC,IAAI,CAoBvB;IA+BD;;;;;OAKG;IACH,uCAEC;IAED;;;;;;;;;;OAUG;IACH,YATW,MAAM,GAAC,MAAM,EAAE;QAEW,SAAS,GAAnC,MAAM,GAAC,MAAM,EAAE;QACG,OAAO,GAAzB,OAAO;QACW,UAAU,GAA5B,OAAO;QACU,UAAU,GAA3B,MAAM;QACY,aAAa,GAA/B,OAAO;;mBACQ,MAAM,EAAE,GAAC,MAAM,EAAE,EAAE,GAAC,MAAM;wBAAkB,GAAG,EAAE,GAAC,MAAM;MA2JjF;IAED;;;;;OAKG;IACH,mBAHW,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,GAAC,IAAI,CAgCzB;IAED;;;;;;OAMG;IACH,aAJW,MAAM,cACN,MAAM,GAAC,IAAI,GACT,MAAM,EAAE,CAapB;IAED;;;;;OAKG;IACH,oBAJW,MAAM,EAAE,EAAE,sBAER,MAAM,EAAE,CAIpB;IAED;;;;;;;;;;OAUG;IACH,kBARW,MAAM,EAAE;QAEc,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;QAEL,MAAM,CAYlB;IAED;;;;;;;;OAQG;IACH,yBAPW,MAAM,EAAE;QAEc,mBAAmB,GAAzC,OAAO;QACe,4BAA4B,GAAlD,OAAO;QAEL,MAAM,CAgClB;CAEJ;AAgCD;;;GAGG;AACH;CAKC;AACD;;;GAGG;AACH;CAKC;AACD;CAKC;AACD;CAKC;AACD;CAKC;AACD;CAKC;AACD;CAAgE;AAEhE;CAAwD;AACxD;CAA0D;AAC1D;CAA0D;AAC1D;CAA6D;AAE7D;CAA2D;AAC3D;CAA2D;AAE3D;CAAgE;AAChE;CAA2D;AAE3D;CAKC;AAED;CAA6D;AAE7D;;;;;;;;;;;;GAYG;AACH;IAEI,sDAKC;IAFG,sBAA+C;IAC/C,sBAAiF;IAGrF;;;;;;OAMG;IACH,sCALW,MAAM,GAAC,MAAM,EAAE,qDAkCzB;CACJ;AA6HD;;;GAGG;AACH;IAEI;;;;;OAKG;IACH,uBAJW,MAAM;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAC,CAAC;iBAE/C,SAAS,GAAC,MAAM;YAAC,QAAQ,EAAE,MAAM,GAAC,IAAI,CAAC;YAAC,SAAS,EAAE,MAAM,MAAM,GAAC,IAAI,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAC,CAAC;SA8RxH;IAED;;;;;;OAMG;IACH,kCAsFC;IAED,eAAe;IACf,8BAgBC;IAED;;;;;;;;;OASG;IACH,+BAcC;IAqBD;;;;OAIG;IACH,6BA2BC;IAED;;;;;OAKG;IACH,6BAqCC;IAED;;;;OAIG;IACH,4BAoCC;IAED;;;;;;;;OAQG;IACH,0BA+CC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH;QAV4B,QAAQ,GAAzB,MAAM;QAIW,IAAI,GAArB,MAAM;QAGY,aAAa,GAA/B,OAAO;QACL,MAAM,EAAE,EAAE,CA0EtB;CACJ;AACD;CAA6D;AAC7D;CAA0D;AAG1D;;;GAGG;AACH;IASQ,sBAAqC;IAErC,mCAEC;IAKL;;;;;;;OAOG;IACH,mBAHW,MAAM,GAAC,IAAI,SAsBrB;CAEJ;AAED;;;;;;GAMG;AACH;IACI;;;;;;;;;;;;;;;;;;;;;;;;MA0BC;IAGD;;;;;;;;;;;;;;OAcG;IACH,sDATW,MAAM,qFAKN,iBAAiB,GAEf,QAAQ,mBAAmB,CAAC,CA6BxC;CACJ;gCAz6GY,OAAO,gBAAgB,EAAE,iBAAiB;;;;;WA0azC,MAAM;;;;UACN,MAAM;;;;YACN,MAAM;;;;WACN,OAAO;;;;WACP,OAAO;;qCA6rBR,SAAS,GAAC,UAAU,GAAC,oBAAoB,GAAC,gBAAgB,GAAC,YAAY;;;;;AAjepF;;;GAGG;AACH;IASI;;;;;;OAMG;IACH,gCAHa,UAAU,CA6BtB;IAzCD;;OAEG;IACH,yBAGC;IADG,YAAoB;IAsCxB;;;;;;OAMG;IACH,gBAJW,MAAM,GACJ,MAAM,CAKlB;IAED;;;;OAIG;IACH,YAHW,MAAM,GACJ,MAAM,CAIlB;CAEJ;;;;;AAwPD;;;;GAIG;AACH;IACI;;;;;;;KAOC;IACD,gCAHW,YAAY,CA2BtB;IAED;;;;;;;KAOC;IACD,wBAJS,MAAM,GACJ,MAAM,EAAE,CAKlB;IAED;;;;OAIG;IACH,mBAHW,MAAM,GAAC,MAAM,EAAE,GACb,MAAM,EAAE,CAUpB;IAED;;;;OAIG;IACH,YAHW,MAAM,GAAC,MAAM,EAAE,GACb,MAAM,EAAE,CAIpB;CACJ;;;;;AA0LD;;GAEG;AACH;IAUI;;;;;;OAMG;IACH,gCAHa,aAAa,CAkBzB;IA9BD;;OAEG;IACH,yBAGC;IADG,YAAoB;IA2BxB;;;;;;;OAOG;IACH,mDAEC;IAED;;;;;OAKG;IACH,4CAEC;CACJ;;;;;AAkGD;;;GAGG;AACH;IAgBI;;;;;;KAMC;IACD,gCAHW,OAAO,CA2BjB;IA7CD;;;;MAIE;IACF,yBAOC;IALG,YAAoB;IAEpB,oBAAsB;IACtB,wBAA8B;IAC9B,kBAAuC;IAoC3C;;;;;MAKE;IACF,cAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;MAIE;IACF,eAHU,MAAM,EAAE,GACN,MAAM,CAIjB;IAED;;;;;;OAMG;IACH,qBAJW,MAAM,EAAE,GACN,MAAM,EAAE,CAKpB;CAEJ;uBAn8CsB,mBAAmB"}