/** * Expr-lang lexer/tokenizer * Handles: identifiers, numbers, strings, bytes literals, operators, comments */ import { Token, TokenKind, EOF_TOKEN } from './types'; // Tokens that should be treated as Operators even if they look like identifiers const OPERATOR_TOKENS = new Set([ 'let', 'if', 'else', 'not', 'and', 'or', 'in', 'matches', 'contains', 'startsWith', 'endsWith', 'hasPrefix', 'hasSuffix', ]); export class Lexer { private input = ''; private pos = 0; private line = 1; private column = 1; private startLine = 1; private startColumn = 1; reset(input: string): void { this.input = input; this.pos = 0; this.line = 1; this.column = 1; this.startLine = 1; this.startColumn = 1; } next(): Token { this.skipWhitespace(); this.skipComments(); if (this.pos >= this.input.length) { return { ...EOF_TOKEN, start: this.pos, end: this.pos, line: this.line, column: this.column }; } this.startLine = this.line; this.startColumn = this.column; const ch = this.input[this.pos]; // String literals if (ch === '"' || ch === "'") { return this.readString(ch); } if (ch === '`') { return this.readBacktickString(); } // Byte literals: b"..." or b'...' or B"..." or B'...' if ((ch === 'b' || ch === 'B') && this.pos + 1 < this.input.length) { const next = this.input[this.pos + 1]; if (next === '"' || next === "'") { return this.readByteLiteral(); } } // Numbers if (this.isDigit(ch) || (ch === '.' && this.pos + 1 < this.input.length && this.isDigit(this.input[this.pos + 1]))) { return this.readNumber(); } // Identifiers and keywords if (this.isIdentStart(ch)) { const token = this.readIdentOrKeyword(); return token; } // Operators and brackets return this.readOperatorOrBracket(); } private skipWhitespace(): void { while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (ch === ' ' || ch === '\t' || ch === '\r') { this.pos++; this.column++; } else if (ch === '\n') { this.pos++; this.line++; this.column = 1; } else { break; } } } private skipComments(): void { while (this.pos < this.input.length) { // Line comment // if (this.input[this.pos] === '/' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '/') { this.pos += 2; this.column += 2; while (this.pos < this.input.length && this.input[this.pos] !== '\n') { this.pos++; this.column++; } this.skipWhitespace(); continue; } // Block comment /* */ if (this.input[this.pos] === '/' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '*') { this.pos += 2; this.column += 2; while (this.pos < this.input.length) { if (this.input[this.pos] === '*' && this.pos + 1 < this.input.length && this.input[this.pos + 1] === '/') { this.pos += 2; this.column += 2; break; } if (this.input[this.pos] === '\n') { this.line++; this.column = 1; } else { this.column++; } this.pos++; } this.skipWhitespace(); continue; } break; } } private readString(quote: string): Token { const start = this.pos; this.pos++; // skip opening quote this.column++; let value = ''; while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (ch === '\\') { if (this.pos + 1 >= this.input.length) break; const next = this.input[this.pos + 1]; switch (next) { case 'n': value += '\n'; break; case 't': value += '\t'; break; case 'r': value += '\r'; break; case '\\': value += '\\'; break; case quote: value += quote; break; case 'x': { const hex = this.input.substr(this.pos + 2, 2); value += String.fromCharCode(parseInt(hex, 16)); this.pos += 2; this.column += 2; break; } case 'u': { const unicode = this.input.substr(this.pos + 2, 4); value += String.fromCharCode(parseInt(unicode, 16)); this.pos += 2; this.column += 2; break; } default: value += next; } this.pos += 2; this.column += 2; } else if (ch === quote) { this.pos++; // skip closing quote this.column++; return this.makeToken(TokenKind.String, value, start); } else { value += ch; this.pos++; this.column++; if (ch === '\n') { this.line++; this.column = 1; } } } return this.makeToken(TokenKind.String, value, start); } private readBacktickString(): Token { const start = this.pos; this.pos++; // skip opening backtick this.column++; let value = ''; while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (ch === '`') { this.pos++; this.column++; return this.makeToken(TokenKind.String, value, start); } value += ch; this.pos++; if (ch === '\n') { this.line++; this.column = 1; } else { this.column++; } } return this.makeToken(TokenKind.String, value, start); } private readByteLiteral(): Token { // b"..." or b'...' or B"..." or B'...' const start = this.pos; this.pos++; // skip 'b' or 'B' this.column++; const quote = this.input[this.pos]; this.pos++; // skip quote this.column++; let value = ''; while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (ch === '\\') { if (this.pos + 1 >= this.input.length) break; const next = this.input[this.pos + 1]; switch (next) { case 'n': value += '\n'; break; case 't': value += '\t'; break; case 'r': value += '\r'; break; case '\\': value += '\\'; break; case quote: value += quote; break; case 'x': { const hex = this.input.substr(this.pos + 2, 2); value += String.fromCharCode(parseInt(hex, 16)); this.pos += 2; this.column += 2; break; } default: // octal escape \NNN if (this.isDigit(next)) { const octal = this.input.substr(this.pos + 1, 3); value += String.fromCharCode(parseInt(octal, 8)); this.pos += 2; this.column += 2; break; } value += next; } this.pos += 2; this.column += 2; } else if (ch === quote) { this.pos++; this.column++; return this.makeToken(TokenKind.String, value, start); } else { value += ch; this.pos++; this.column++; } } return this.makeToken(TokenKind.String, value, start); } private readNumber(): Token { const start = this.pos; let value = ''; while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (this.isDigit(ch) || ch === '.' || ch === 'e' || ch === 'E' || ch === '+' || ch === '-' || ch === '_' || ch === 'x' || ch === 'X' || ch === 'o' || ch === 'O' || ch === 'b' || ch === 'B' || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')) { value += ch; this.pos++; this.column++; } else { break; } } return this.makeToken(TokenKind.Number, value, start); } private readIdentOrKeyword(): Token { const start = this.pos; let value = ''; while (this.pos < this.input.length) { const ch = this.input[this.pos]; if (this.isIdentPart(ch)) { value += ch; this.pos++; this.column++; } else { break; } } // Keywords that are operators if (OPERATOR_TOKENS.has(value)) { return this.makeToken(TokenKind.Operator, value, start); } return this.makeToken(TokenKind.Identifier, value, start); } private readOperatorOrBracket(): Token { const start = this.pos; const ch = this.input[this.pos]; // Brackets if ('()[]{}'.includes(ch)) { this.pos++; this.column++; return this.makeToken(TokenKind.Bracket, ch, start); } // Multi-character operators const twoChar = this.input.substr(this.pos, 2); const threeChar = this.input.substr(this.pos, 3); // 3-character operators: **= // 2-character operators const twoCharOps = ['==', '!=', '<=', '>=', '&&', '||', '??', '?.', '..', '**', '//', '::', '->']; const threeCharOps = ['...', '<<=', '>>=']; if (threeCharOps.includes(threeChar)) { this.pos += 3; this.column += 3; return this.makeToken(TokenKind.Operator, threeChar, start); } if (twoCharOps.includes(twoChar)) { this.pos += 2; this.column += 2; return this.makeToken(TokenKind.Operator, twoChar, start); } // Single-character operators and separators const singleCharOps = '+-*/%^=<>!&|?.,;:#@$~'; if (singleCharOps.includes(ch)) { this.pos++; this.column++; return this.makeToken(TokenKind.Operator, ch, start); } // Unrecognized character this.pos++; this.column++; return this.makeToken(TokenKind.Operator, ch, start); } private isDigit(ch: string): boolean { return ch >= '0' && ch <= '9'; } private isIdentStart(ch: string): boolean { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_' || ch === '$' || isUnicodeLetter(ch); } private isIdentPart(ch: string): boolean { return this.isIdentStart(ch) || this.isDigit(ch) || isUnicodeDigit(ch); } private makeToken(kind: TokenKind, value: string, start: number): Token { return { kind, value, start, end: this.pos, line: this.startLine, column: this.startColumn, }; } } /** Check if a character is a Unicode letter (including Chinese, Japanese, etc.) */ function isUnicodeLetter(ch: string): boolean { return /\p{L}/u.test(ch); } /** Check if a character is a Unicode digit */ function isUnicodeDigit(ch: string): boolean { return /\p{N}/u.test(ch); }