{"version":3,"file":"tokenizer-1hAHZzmp.mjs","names":["#source","#buffer","#pos","#scanNext"],"sources":["../src/tokenizer.ts"],"sourcesContent":["export type TokenKind =\n  | 'Ident'\n  | 'StringLiteral'\n  | 'NumberLiteral'\n  | 'At'\n  | 'DoubleAt'\n  | 'LBrace'\n  | 'RBrace'\n  | 'LParen'\n  | 'RParen'\n  | 'LBracket'\n  | 'RBracket'\n  | 'Equals'\n  | 'Question'\n  | 'Dot'\n  | 'Comma'\n  | 'Colon'\n  | 'Whitespace'\n  | 'Newline'\n  | 'Comment'\n  | 'Invalid'\n  | 'Eof';\n\nexport interface Token {\n  readonly kind: TokenKind;\n  readonly text: string;\n}\n\nexport class Tokenizer {\n  readonly #source: string;\n  #pos: number;\n  readonly #buffer: Token[];\n\n  constructor(source: string) {\n    this.#source = source;\n    this.#pos = 0;\n    this.#buffer = [];\n  }\n\n  next(): Token {\n    const next = this.#buffer.shift();\n    if (next) {\n      return next;\n    }\n    return this.#scanNext();\n  }\n\n  peek(offset = 0): Token {\n    if (offset > this.#buffer.length) {\n      const last = this.#buffer.at(-1);\n      if (last?.kind === 'Eof') {\n        return last;\n      }\n    }\n\n    const token = this.#buffer[offset];\n    if (token) {\n      return token;\n    }\n\n    while (this.#buffer.length <= offset) {\n      const token = this.#scanNext();\n      if (token.kind === 'Eof') {\n        return token;\n      }\n      this.#buffer.push(token);\n    }\n\n    return this.#buffer[offset] as Token;\n  }\n\n  #scanNext(): Token {\n    const token = scan(this.#source, this.#pos);\n    this.#pos += token.text.length;\n    return token;\n  }\n}\n\nfunction scan(source: string, pos: number): Token {\n  if (pos >= source.length) {\n    return { kind: 'Eof', text: '' };\n  }\n\n  return (\n    scanNewline(source, pos) ??\n    scanWhitespace(source, pos) ??\n    scanComment(source, pos) ??\n    scanAt(source, pos) ??\n    scanKeywordNumber(source, pos) ??\n    scanIdent(source, pos) ??\n    scanNumber(source, pos) ??\n    scanString(source, pos) ??\n    scanPunctuation(source, pos) ?? {\n      kind: 'Invalid' as const,\n      text: readChar(source, pos),\n    }\n  );\n}\n\nfunction scanNewline(source: string, pos: number): Token | undefined {\n  const ch = source.charAt(pos);\n  if (ch !== '\\r' && ch !== '\\n') return undefined;\n  if (ch === '\\r' && source.charAt(pos + 1) === '\\n') {\n    return { kind: 'Newline', text: '\\r\\n' };\n  }\n  return { kind: 'Newline', text: ch };\n}\n\nfunction scanWhitespace(source: string, pos: number): Token | undefined {\n  const ch = source.charAt(pos);\n  if (ch !== ' ' && ch !== '\\t') return undefined;\n  let end = pos + 1;\n  while (end < source.length) {\n    const c = source.charAt(end);\n    if (c !== ' ' && c !== '\\t') break;\n    end++;\n  }\n  return { kind: 'Whitespace', text: source.slice(pos, end) };\n}\n\nfunction scanComment(source: string, pos: number): Token | undefined {\n  if (source.charAt(pos) !== '/' || source.charAt(pos + 1) !== '/') return undefined;\n  let end = pos + 2;\n  while (end < source.length) {\n    const c = source.charAt(end);\n    if (c === '\\n' || c === '\\r') break;\n    end++;\n  }\n  return { kind: 'Comment', text: source.slice(pos, end) };\n}\n\nfunction scanAt(source: string, pos: number): Token | undefined {\n  if (source.charAt(pos) !== '@') return undefined;\n  if (source.charAt(pos + 1) === '@') {\n    return { kind: 'DoubleAt', text: '@@' };\n  }\n  return { kind: 'At', text: '@' };\n}\n\nfunction scanIdent(source: string, pos: number): Token | undefined {\n  const ch = readChar(source, pos);\n  if (!isIdentStart(ch)) return undefined;\n  let end = pos + ch.length;\n  while (end < source.length) {\n    const c = readChar(source, end);\n    if (isIdentPart(c)) {\n      end += c.length;\n    } else {\n      break;\n    }\n  }\n  return { kind: 'Ident', text: source.slice(pos, end) };\n}\n\nconst KEYWORD_NUMBERS = ['-Infinity', 'Infinity', 'NaN'] as const;\n\n// `NaN`, `Infinity`, and `-Infinity` are numeric literals; they must match\n// before `scanIdent` and are word-bounded, so `Infinityx` stays an `Ident`.\nfunction scanKeywordNumber(source: string, pos: number): Token | undefined {\n  for (const word of KEYWORD_NUMBERS) {\n    if (!source.startsWith(word, pos)) continue;\n    const after = readChar(source, pos + word.length);\n    if (after !== '' && isIdentPart(after)) return undefined;\n    return { kind: 'NumberLiteral', text: word };\n  }\n  return undefined;\n}\n\nfunction scanNumber(source: string, pos: number): Token | undefined {\n  let end = pos;\n  if (source.charAt(end) === '-') {\n    if (end + 1 >= source.length || !isDigit(source.charAt(end + 1))) return undefined;\n    end++;\n  } else if (!isDigit(source.charAt(end))) {\n    return undefined;\n  }\n  end++;\n  while (end < source.length && isDigit(source.charAt(end))) {\n    end++;\n  }\n  if (source.charAt(end) === '.' && end + 1 < source.length && isDigit(source.charAt(end + 1))) {\n    end++;\n    while (end < source.length && isDigit(source.charAt(end))) {\n      end++;\n    }\n  }\n  return { kind: 'NumberLiteral', text: source.slice(pos, end) };\n}\n\nfunction scanString(source: string, pos: number): Token | undefined {\n  const quote = source.charAt(pos);\n  if (quote !== '\"' && quote !== \"'\") return undefined;\n  let end = pos + 1;\n  while (end < source.length) {\n    const c = source.charAt(end);\n    if (c === '\\\\' && end + 1 < source.length) {\n      end += 2;\n      continue;\n    }\n    if (c === quote) {\n      end++;\n      return { kind: 'StringLiteral', text: source.slice(pos, end) };\n    }\n    if (c === '\\n' || c === '\\r') {\n      // Unterminated string: stop before the newline.\n      return { kind: 'StringLiteral', text: source.slice(pos, end) };\n    }\n    end++;\n  }\n  return { kind: 'StringLiteral', text: source.slice(pos, end) };\n}\n\n/**\n * `scanString` emits the same `StringLiteral` kind for well-formed and\n * unterminated strings, so callers ask here to tell them apart.\n *\n * The closing quote is unescaped iff an **even** number of backslashes precede\n * it (each `\\\\` pair cancels; an odd run leaves the final `\\` escaping the\n * quote), so counting the trailing backslash run suffices — no full re-scan:\n *\n * - `\"ok\"`  → 0 backslashes (even) → terminated\n * - `'a\\'`  → 1 backslash  (odd)   → the quote is escaped → unterminated\n * - `\"a\\\\\"` → 2 backslashes (even) → escaped `\\`, real `\"` → terminated\n */\nexport function isTerminatedStringLiteral(text: string): boolean {\n  const quote = text.charAt(0);\n  if (quote !== '\"' && quote !== \"'\") return false;\n  if (text.length < 2 || text.charAt(text.length - 1) !== quote) {\n    return false;\n  }\n  let backslashes = 0;\n  for (let i = text.length - 2; i >= 1 && text.charAt(i) === '\\\\'; i--) {\n    backslashes++;\n  }\n  return backslashes % 2 === 0;\n}\n\nfunction scanPunctuation(source: string, pos: number): Token | undefined {\n  const kind = PUNCTUATION[source.charAt(pos)];\n  if (kind === undefined) return undefined;\n  return { kind, text: source.charAt(pos) };\n}\n\nfunction readChar(source: string, pos: number): string {\n  const cp = source.codePointAt(pos);\n  return cp !== undefined ? String.fromCodePoint(cp) : '';\n}\n\nfunction isIdentStart(ch: string): boolean {\n  return /\\p{L}/u.test(ch) || ch === '_';\n}\n\nfunction isIdentPart(ch: string): boolean {\n  return isIdentStart(ch) || isDigit(ch) || ch === '-';\n}\n\nfunction isDigit(ch: string): boolean {\n  return ch >= '0' && ch <= '9';\n}\n\nconst PUNCTUATION: Record<string, TokenKind> = {\n  '{': 'LBrace',\n  '}': 'RBrace',\n  '(': 'LParen',\n  ')': 'RParen',\n  '[': 'LBracket',\n  ']': 'RBracket',\n  '=': 'Equals',\n  '?': 'Question',\n  '.': 'Dot',\n  ',': 'Comma',\n  ':': 'Colon',\n};\n"],"mappings":";AA4BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CAEA,YAAY,QAAgB;EAC1B,KAAKA,UAAU;EACf,KAAKE,OAAO;EACZ,KAAKD,UAAU,CAAC;CAClB;CAEA,OAAc;EACZ,MAAM,OAAO,KAAKA,QAAQ,MAAM;EAChC,IAAI,MACF,OAAO;EAET,OAAO,KAAKE,UAAU;CACxB;CAEA,KAAK,SAAS,GAAU;EACtB,IAAI,SAAS,KAAKF,QAAQ,QAAQ;GAChC,MAAM,OAAO,KAAKA,QAAQ,GAAG,EAAE;GAC/B,IAAI,MAAM,SAAS,OACjB,OAAO;EAEX;EAEA,MAAM,QAAQ,KAAKA,QAAQ;EAC3B,IAAI,OACF,OAAO;EAGT,OAAO,KAAKA,QAAQ,UAAU,QAAQ;GACpC,MAAM,QAAQ,KAAKE,UAAU;GAC7B,IAAI,MAAM,SAAS,OACjB,OAAO;GAET,KAAKF,QAAQ,KAAK,KAAK;EACzB;EAEA,OAAO,KAAKA,QAAQ;CACtB;CAEA,YAAmB;EACjB,MAAM,QAAQ,KAAK,KAAKD,SAAS,KAAKE,IAAI;EAC1C,KAAKA,QAAQ,MAAM,KAAK;EACxB,OAAO;CACT;AACF;AAEA,SAAS,KAAK,QAAgB,KAAoB;CAChD,IAAI,OAAO,OAAO,QAChB,OAAO;EAAE,MAAM;EAAO,MAAM;CAAG;CAGjC,OACE,YAAY,QAAQ,GAAG,KACvB,eAAe,QAAQ,GAAG,KAC1B,YAAY,QAAQ,GAAG,KACvB,OAAO,QAAQ,GAAG,KAClB,kBAAkB,QAAQ,GAAG,KAC7B,UAAU,QAAQ,GAAG,KACrB,WAAW,QAAQ,GAAG,KACtB,WAAW,QAAQ,GAAG,KACtB,gBAAgB,QAAQ,GAAG,KAAK;EAC9B,MAAM;EACN,MAAM,SAAS,QAAQ,GAAG;CAC5B;AAEJ;AAEA,SAAS,YAAY,QAAgB,KAAgC;CACnE,MAAM,KAAK,OAAO,OAAO,GAAG;CAC5B,IAAI,OAAO,QAAQ,OAAO,MAAM,OAAO,KAAA;CACvC,IAAI,OAAO,QAAQ,OAAO,OAAO,MAAM,CAAC,MAAM,MAC5C,OAAO;EAAE,MAAM;EAAW,MAAM;CAAO;CAEzC,OAAO;EAAE,MAAM;EAAW,MAAM;CAAG;AACrC;AAEA,SAAS,eAAe,QAAgB,KAAgC;CACtE,MAAM,KAAK,OAAO,OAAO,GAAG;CAC5B,IAAI,OAAO,OAAO,OAAO,KAAM,OAAO,KAAA;CACtC,IAAI,MAAM,MAAM;CAChB,OAAO,MAAM,OAAO,QAAQ;EAC1B,MAAM,IAAI,OAAO,OAAO,GAAG;EAC3B,IAAI,MAAM,OAAO,MAAM,KAAM;EAC7B;CACF;CACA,OAAO;EAAE,MAAM;EAAc,MAAM,OAAO,MAAM,KAAK,GAAG;CAAE;AAC5D;AAEA,SAAS,YAAY,QAAgB,KAAgC;CACnE,IAAI,OAAO,OAAO,GAAG,MAAM,OAAO,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,OAAO,KAAA;CACzE,IAAI,MAAM,MAAM;CAChB,OAAO,MAAM,OAAO,QAAQ;EAC1B,MAAM,IAAI,OAAO,OAAO,GAAG;EAC3B,IAAI,MAAM,QAAQ,MAAM,MAAM;EAC9B;CACF;CACA,OAAO;EAAE,MAAM;EAAW,MAAM,OAAO,MAAM,KAAK,GAAG;CAAE;AACzD;AAEA,SAAS,OAAO,QAAgB,KAAgC;CAC9D,IAAI,OAAO,OAAO,GAAG,MAAM,KAAK,OAAO,KAAA;CACvC,IAAI,OAAO,OAAO,MAAM,CAAC,MAAM,KAC7B,OAAO;EAAE,MAAM;EAAY,MAAM;CAAK;CAExC,OAAO;EAAE,MAAM;EAAM,MAAM;CAAI;AACjC;AAEA,SAAS,UAAU,QAAgB,KAAgC;CACjE,MAAM,KAAK,SAAS,QAAQ,GAAG;CAC/B,IAAI,CAAC,aAAa,EAAE,GAAG,OAAO,KAAA;CAC9B,IAAI,MAAM,MAAM,GAAG;CACnB,OAAO,MAAM,OAAO,QAAQ;EAC1B,MAAM,IAAI,SAAS,QAAQ,GAAG;EAC9B,IAAI,YAAY,CAAC,GACf,OAAO,EAAE;OAET;CAEJ;CACA,OAAO;EAAE,MAAM;EAAS,MAAM,OAAO,MAAM,KAAK,GAAG;CAAE;AACvD;AAEA,MAAM,kBAAkB;CAAC;CAAa;CAAY;AAAK;AAIvD,SAAS,kBAAkB,QAAgB,KAAgC;CACzE,KAAK,MAAM,QAAQ,iBAAiB;EAClC,IAAI,CAAC,OAAO,WAAW,MAAM,GAAG,GAAG;EACnC,MAAM,QAAQ,SAAS,QAAQ,MAAM,KAAK,MAAM;EAChD,IAAI,UAAU,MAAM,YAAY,KAAK,GAAG,OAAO,KAAA;EAC/C,OAAO;GAAE,MAAM;GAAiB,MAAM;EAAK;CAC7C;AAEF;AAEA,SAAS,WAAW,QAAgB,KAAgC;CAClE,IAAI,MAAM;CACV,IAAI,OAAO,OAAO,GAAG,MAAM,KAAK;EAC9B,IAAI,MAAM,KAAK,OAAO,UAAU,CAAC,QAAQ,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,OAAO,KAAA;EACzE;CACF,OAAO,IAAI,CAAC,QAAQ,OAAO,OAAO,GAAG,CAAC,GACpC;CAEF;CACA,OAAO,MAAM,OAAO,UAAU,QAAQ,OAAO,OAAO,GAAG,CAAC,GACtD;CAEF,IAAI,OAAO,OAAO,GAAG,MAAM,OAAO,MAAM,IAAI,OAAO,UAAU,QAAQ,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG;EAC5F;EACA,OAAO,MAAM,OAAO,UAAU,QAAQ,OAAO,OAAO,GAAG,CAAC,GACtD;CAEJ;CACA,OAAO;EAAE,MAAM;EAAiB,MAAM,OAAO,MAAM,KAAK,GAAG;CAAE;AAC/D;AAEA,SAAS,WAAW,QAAgB,KAAgC;CAClE,MAAM,QAAQ,OAAO,OAAO,GAAG;CAC/B,IAAI,UAAU,QAAO,UAAU,KAAK,OAAO,KAAA;CAC3C,IAAI,MAAM,MAAM;CAChB,OAAO,MAAM,OAAO,QAAQ;EAC1B,MAAM,IAAI,OAAO,OAAO,GAAG;EAC3B,IAAI,MAAM,QAAQ,MAAM,IAAI,OAAO,QAAQ;GACzC,OAAO;GACP;EACF;EACA,IAAI,MAAM,OAAO;GACf;GACA,OAAO;IAAE,MAAM;IAAiB,MAAM,OAAO,MAAM,KAAK,GAAG;GAAE;EAC/D;EACA,IAAI,MAAM,QAAQ,MAAM,MAEtB,OAAO;GAAE,MAAM;GAAiB,MAAM,OAAO,MAAM,KAAK,GAAG;EAAE;EAE/D;CACF;CACA,OAAO;EAAE,MAAM;EAAiB,MAAM,OAAO,MAAM,KAAK,GAAG;CAAE;AAC/D;;;;;;;;;;;;;AAcA,SAAgB,0BAA0B,MAAuB;CAC/D,MAAM,QAAQ,KAAK,OAAO,CAAC;CAC3B,IAAI,UAAU,QAAO,UAAU,KAAK,OAAO;CAC3C,IAAI,KAAK,SAAS,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,MAAM,OACtD,OAAO;CAET,IAAI,cAAc;CAClB,KAAK,IAAI,IAAI,KAAK,SAAS,GAAG,KAAK,KAAK,KAAK,OAAO,CAAC,MAAM,MAAM,KAC/D;CAEF,OAAO,cAAc,MAAM;AAC7B;AAEA,SAAS,gBAAgB,QAAgB,KAAgC;CACvE,MAAM,OAAO,YAAY,OAAO,OAAO,GAAG;CAC1C,IAAI,SAAS,KAAA,GAAW,OAAO,KAAA;CAC/B,OAAO;EAAE;EAAM,MAAM,OAAO,OAAO,GAAG;CAAE;AAC1C;AAEA,SAAS,SAAS,QAAgB,KAAqB;CACrD,MAAM,KAAK,OAAO,YAAY,GAAG;CACjC,OAAO,OAAO,KAAA,IAAY,OAAO,cAAc,EAAE,IAAI;AACvD;AAEA,SAAS,aAAa,IAAqB;CACzC,OAAO,SAAS,KAAK,EAAE,KAAK,OAAO;AACrC;AAEA,SAAS,YAAY,IAAqB;CACxC,OAAO,aAAa,EAAE,KAAK,QAAQ,EAAE,KAAK,OAAO;AACnD;AAEA,SAAS,QAAQ,IAAqB;CACpC,OAAO,MAAM,OAAO,MAAM;AAC5B;AAEA,MAAM,cAAyC;CAC7C,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;AACP"}