{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../../src/core/services/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,CAAC;IACrB,iBAAiB,CAAC,EAAE,cAAc,EAAE,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,SAAS,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAC/B,SAAS,EAAE,SAAS,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAgBtE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,SAAS,UAAQ,GAChB,SAAS,CAMX;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC5B,MAAM,CAQR;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,EAAE,CA0CjE;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQ7C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,MAAM,EACzB,SAAS,GAAE,MAAwC,GAClD,oBAAoB,CAoDtB;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,SAAS,GAAE,MAAwC,EACnD,oBAAoB,CAAC,EAAE,cAAc,EAAE,GACtC,cAAc,CAqFhB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAwC,GAClD,aAAa,CAwFf;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,GACf;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,OAAO,CAAC;CACzB,CAgBA;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EACpC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAC5B;IAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAcxD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,UAAU,EAAE;IAAE,cAAc,EAAE,OAAO,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACjF,MAAM,GAAG,SAAS,CAKpB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,CAAC,EACzC,KAAK,EAAE,CAAC,EAAE,EACV,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EACjC,IAAI,SAAI,EACR,QAAQ,SAAK,GACZ;IACD,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,SAAS,EAAE,SAAS,CAAC;IACrB,UAAU,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC;CACH,CAsCA"}