import { TokenKind } from './types'; import type { Token } from './types'; // ─── Character helpers ──────────────────────────────────────────── const SPECIAL_CHARS = '\\():<>"*{}'; /** Whitespace per the grammar's Space rule. */ export function isSpaceChar(ch: string): boolean { return ch === ' ' || ch === '\t' || ch === '\r' || ch === '\n' || ch === '\u00A0'; } function isSpecialChar(ch: string): boolean { return ch.length === 1 && SPECIAL_CHARS.includes(ch); } function isHexDigit(ch: string): boolean { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); } function isHex4(input: string, pos: number): boolean { if (pos + 4 > input.length) return false; for (let i = pos; i < pos + 4; i++) { if (!isHexDigit(input[i])) return false; } return true; } function startsWithCi(input: string, pos: number, word: string): boolean { return input.slice(pos, pos + word.length).toLowerCase() === word; } function isSpaceAt(input: string, pos: number): boolean { return pos < input.length && isSpaceChar(input[pos]); } /** * Mirror of the grammar's Keyword rule, used as !Keyword inside UnquotedCharacter: * Space 'or'i Space / Space 'and'i Space / 'not'i Space * True when a keyword starts at `pos`, meaning an unquoted literal must stop here. */ function isKeywordBoundaryAt(input: string, pos: number): boolean { if (startsWithCi(input, pos, 'not') && isSpaceAt(input, pos + 3)) return true; if (isSpaceAt(input, pos)) { if (startsWithCi(input, pos + 1, 'or') && isSpaceAt(input, pos + 3)) return true; if (startsWithCi(input, pos + 1, 'and') && isSpaceAt(input, pos + 4)) return true; } return false; } /** * Decide whether the token starting at `pos` is an and/or/not keyword. * and/or must be surrounded by whitespace; not only needs whitespace after. */ function keywordAtStart(input: string, pos: number): TokenKind | null { if (startsWithCi(input, pos, 'and') && isSpaceAt(input, pos + 3) && pos > 0 && isSpaceAt(input, pos - 1)) return TokenKind.And; if (startsWithCi(input, pos, 'or') && isSpaceAt(input, pos + 2) && pos > 0 && isSpaceAt(input, pos - 1)) return TokenKind.Or; if (startsWithCi(input, pos, 'not') && isSpaceAt(input, pos + 3)) return TokenKind.Not; return null; } // ─── Escape decoding ────────────────────────────────────────────── function decodeEscape(input: string, backslashPos: number): { text: string; length: number } { const n = input[backslashPos + 1]; if (n === undefined) return { text: '\\', length: 1 }; if (n === 't') return { text: '\t', length: 2 }; if (n === 'r') return { text: '\r', length: 2 }; if (n === 'n') return { text: '\n', length: 2 }; if (n === 'u' && isHex4(input, backslashPos + 2)) { return { text: String.fromCharCode(parseInt(input.slice(backslashPos + 2, backslashPos + 6), 16)), length: 6 }; } if (isSpecialChar(n)) return { text: n, length: 2 }; const three = input.slice(backslashPos + 1, backslashPos + 4).toLowerCase(); if (three === 'and' || three === 'not') return { text: three, length: 4 }; const two = input.slice(backslashPos + 1, backslashPos + 3).toLowerCase(); if (two === 'or') return { text: 'or', length: 3 }; // Tolerate invalid escapes by keeping them verbatim return { text: '\\' + n, length: 2 }; } /** Decode the raw text of an unquoted literal token (escapes resolved, whitespace kept). */ export function decodeUnquotedLiteral(raw: string): string { let out = ''; let i = 0; while (i < raw.length) { const ch = raw[i]; if (ch === '\\') { const { text, length } = decodeEscape(raw, i); out += text; i += length; continue; } out += ch; i++; } return out; } /** Decode the raw text of a quoted string token (quotes stripped, escapes resolved). */ export function decodeQuotedString(raw: string): string { let end = raw.length; if (end >= 2 && raw.endsWith('"')) end = raw.length - 1; const inner = raw.slice(1, end); let out = ''; let i = 0; while (i < inner.length) { const ch = inner[i]; if (ch === '\\' && i + 1 < inner.length) { const n = inner[i + 1]; if (n === 't') { out += '\t'; i += 2; continue; } if (n === 'r') { out += '\r'; i += 2; continue; } if (n === 'n') { out += '\n'; i += 2; continue; } if (n === 'u' && isHex4(inner, i + 2)) { out += String.fromCharCode(parseInt(inner.slice(i + 2, i + 6), 16)); i += 6; continue; } if (n === '\\' || n === '"') { out += n; i += 2; continue; } // Unknown escape: the grammar keeps the backslash as a plain character out += ch; i += 1; continue; } out += ch; i++; } return out; } // ─── Tokenizer ──────────────────────────────────────────────────── /** * Convert an offset (0-based) to Monaco line/column. * Monaco line/column are 1-based. */ export function offsetToPosition(input: string, offset: number): { lineNumber: number; column: number } { let line = 1; let col = 1; for (let i = 0; i < offset && i < input.length; i++) { if (input[i] === '\n') { line++; col = 1; } else { col++; } } return { lineNumber: line, column: col }; } export function tokenize(input: string): Token[] { const tokens: Token[] = []; let pos = 0; const len = input.length; while (pos < len) { const start = pos; const ch = input[pos]; // ── Whitespace ── if (isSpaceChar(ch)) { while (pos < len && isSpaceChar(input[pos])) pos++; tokens.push({ kind: TokenKind.Whitespace, value: input.slice(start, pos), start, end: pos }); continue; } // ── Quoted string ── if (ch === '"') { pos++; while (pos < len) { const c = input[pos]; if (c === '\\') { pos = Math.min(pos + 2, len); continue; } pos++; if (c === '"') break; } tokens.push({ kind: TokenKind.String, value: input.slice(start, pos), start, end: pos }); continue; } // ── Brackets & operators ── if (ch === '(') { pos++; tokens.push({ kind: TokenKind.LParen, value: '(', start, end: pos }); continue; } if (ch === ')') { pos++; tokens.push({ kind: TokenKind.RParen, value: ')', start, end: pos }); continue; } if (ch === '{') { pos++; tokens.push({ kind: TokenKind.LCurly, value: '{', start, end: pos }); continue; } if (ch === '}') { pos++; tokens.push({ kind: TokenKind.RCurly, value: '}', start, end: pos }); continue; } if (ch === ':') { pos++; tokens.push({ kind: TokenKind.Colon, value: ':', start, end: pos }); continue; } if (ch === '<') { if (input[pos + 1] === '=') pos += 2; else pos += 1; tokens.push({ kind: input[start + 1] === '=' ? TokenKind.Lte : TokenKind.Lt, value: input.slice(start, pos), start, end: pos }); continue; } if (ch === '>') { if (input[pos + 1] === '=') pos += 2; else pos += 1; tokens.push({ kind: input[start + 1] === '=' ? TokenKind.Gte : TokenKind.Gt, value: input.slice(start, pos), start, end: pos }); continue; } // ── Keywords (and/or/not with whitespace boundaries) ── const keyword = keywordAtStart(input, pos); if (keyword !== null) { const width = keyword === TokenKind.Or ? 2 : 3; tokens.push({ kind: keyword, value: input.slice(pos, pos + width), start: pos, end: pos + width }); pos += width; continue; } // ── Unquoted literal ── while (pos < len) { if (isKeywordBoundaryAt(input, pos)) break; const c = input[pos]; // An escape pair (\: \( \and \uXXXX ...) is consumed as a unit if (c === '\\') { pos = Math.min(pos + decodeEscape(input, pos).length, len); continue; } if (isSpecialChar(c) && c !== '*') break; pos++; } if (pos === start) pos++; tokens.push({ kind: TokenKind.Ident, value: input.slice(start, pos), start, end: pos }); } tokens.push({ kind: TokenKind.EOF, value: '', start: pos, end: pos }); return tokens; } /** Return tokens with whitespace filtered out. */ export function nonWhitespace(tokens: Token[]): Token[] { return tokens.filter((t) => t.kind !== TokenKind.Whitespace); }