/** * @license * Copyright 2022-2026 Matter.js Authors * SPDX-License-Identifier: Apache-2.0 */ import { FieldValue } from "../common/index.js"; import { BasicToken, Token } from "./Token.js"; function isNameChar(c: string) { return (c >= "A" && c <= "Z") || (c >= "a" && c <= "z") || (c >= "0" && c <= "9") || c === "_"; } /** * DSL lexer. * * Tokenizes simple text dialects. Currently sufficient for Matter conformance and constraint tokenization. */ export class Lexer, const KW extends readonly string[] = []> { #keywords: Set; constructor(keywords?: KW) { if (keywords instanceof Set) { this.#keywords = keywords; } else { this.#keywords = new Set(keywords); } } lex(text: string, error: (code: string, message: string) => void) { return lex(text, error, this.#keywords) as Iterator; } } export namespace Lexer { export const Basic = new Lexer(); } function* lex( text: string, error: (code: string, message: string) => void, keywords: Set, ): Generator { const i = text[Symbol.iterator](); let current = i.next(); if (current.done) { return; } let peeked = i.next(); let line = 1; let char = 1; let startLine = -1; let startChar = -1; function markStart() { startLine = line; startChar = char; } function next() { current = peeked; if (!current.done) { char++; peeked = i.next(); } } function tokenizeName(): Token | BasicToken.Word { const chars = [current.value]; while (peeked.value !== undefined && isNameChar(peeked.value)) { next(); chars.push(current.value); } const name = chars.join(""); if (keywords.has(name)) { return { type: name, startLine, startChar }; } return { type: "word", value: name, startLine, startChar }; } function binaryValueOf(digit: string | undefined) { if (digit === "0") { return 0; } if (digit === "1") { return 1; } } function decimalValueOf(digit: string | undefined) { if (digit === undefined) { return; } if (digit >= "0" && digit <= "9") { return digit.charCodeAt(0) - "0".charCodeAt(0); } } function hexadecimalValueOf(digit: string | undefined) { if (digit === undefined) { return; } if (digit >= "0" && digit <= "9") { return digit.charCodeAt(0) - "0".charCodeAt(0); } if (digit >= "a" && digit <= "f") { return 10 + digit.charCodeAt(0) - "a".charCodeAt(0); } if (digit >= "A" && digit <= "F") { return 10 + digit.charCodeAt(0) - "A".charCodeAt(0); } return; } function tokenizeDigits(base: number, valueOf: (digit: string[1] | undefined) => number | undefined): BasicToken { // The first digit may not actually be a digit if number is hexadecimal or binary const first = valueOf(current.value); if (first === undefined) { error("INVALID_NUMBER", `Expected digit following numeric prefix`); return { type: "value", value: 0, startLine, startChar }; } // Digits accumulate as a bigint because a number states an integer exactly only below 2^53, and the widest // values of the 64 bit types are what the specification writes for their bounds let exact = BigInt(first); const bigBase = BigInt(base); while (true) { const digitValue = valueOf(peeked.value); if (digitValue === undefined) { break; } next(); exact = exact * bigBase + BigInt(digitValue); } // Below the integers a number states without rounding, a bound is a number, which is the shape every reader // already handles. Above them it is a bigint, including where the magnitude happens to be a power of two that // a number would state exactly let num: number | bigint = exact <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(exact) : exact; if (base === 10 && peeked.value === ".") { next(); let fraction = ""; while (true) { const digitValue = valueOf(peeked.value); if (digitValue === undefined) { break; } fraction += peeked.value; next(); } // "…614.0" states the integer before the point, which a fraction's number would round away if (fraction.match(/[1-9]/)) { // Above the integers a number states exactly, a fraction has no form at all: the number rounds to an // integer, and the integer the bigint would state is not the value if (typeof num === "bigint") { error("INVALID_NUMBER", `Fraction of ${exact}.${fraction} states a value no number holds`); } num = Number.parseFloat(`${exact}.${fraction}`); } } // Handle specialized suffices for percents and temperatures if (peeked.value === "%") { next(); return { type: "value", value: FieldValue.Percent(Number(num)), startLine, startChar }; } else if (peeked.value === "°") { next(); if (peeked.value?.toLowerCase() === "c") { next(); } return { type: "value", value: FieldValue.Celsius(Number(num)), startLine, startChar }; } // No special suffix; return raw value return { type: "value", value: num, startLine, startChar }; } while (!current.done) { switch (current.value) { case "|": case "^": case "&": case ".": case ",": case "[": case "]": case "(": case ")": case "{": case "}": case "-": case "+": case "/": case "*": yield { type: current.value, startLine: line, startChar: char }; break; case "0": markStart(); if (current.value === "0") { if (peeked.value === "x") { next(); next(); yield tokenizeDigits(16, hexadecimalValueOf); break; } if (peeked.value === "b") { next(); next(); yield tokenizeDigits(2, binaryValueOf); break; } } yield tokenizeDigits(10, decimalValueOf); break; case "1": case "2": case "3": case "4": case "5": case "6": case "7": case "8": case "9": yield tokenizeDigits(10, decimalValueOf); break; case "!": case ">": case "<": { const base = current.value; if (peeked.value === "=") { const type = `${base}${peeked.value}` as BasicToken.Operator; next(); yield { type, startLine: line, startChar: char, }; } else { yield { type: base, startLine: line, startChar: char }; } } break; case "=": markStart(); if (peeked.value === "=") { next(); } else { error("BAD_EQUAL", `"=" must be followed by another "="`); } yield { type: "==", startLine, startChar }; break; case "\n": line++; char = 0; break; case " ": case "\t": case "\r": case "\v": case "\f": break; default: if (isNameChar(current.value)) { markStart(); yield tokenizeName(); } else { error("GARBAGE_CHARACTER", `Unexpected character "${current.value}"`); } break; } next(); } }