/** * A JSON-compliant tokenizer that turns a utf-8 stream into JSON tokens. * * @example * ```ts * import Tokenizer from "@streamparser/json/tokenizer.js"; * * const tokenizer = new Tokenizer(); * tokenizer.onToken = ({ token, value, offset }) => { * // process the token * }; * * tokenizer.write('{ "test": ["a"] }'); * ``` * * @module */ import { BufferedString, NonBufferedString, type StringBuilder, } from "./utils/bufferedString.js"; import type { ParsedTokenInfo } from "./utils/types/parsedTokenInfo.js"; import TokenType from "./utils/types/tokenType.js"; import { charset, escapedSequences } from "./utils/utf-8.js"; // Tokenizer States const enum TokenizerStates { START, ENDED, ERROR, TRUE1, TRUE2, TRUE3, FALSE1, FALSE2, FALSE3, FALSE4, NULL1, NULL2, NULL3, STRING_DEFAULT, STRING_AFTER_BACKSLASH, STRING_UNICODE_DIGIT_1, STRING_UNICODE_DIGIT_2, STRING_UNICODE_DIGIT_3, STRING_UNICODE_DIGIT_4, STRING_INCOMPLETE_CHAR, NUMBER_AFTER_INITIAL_MINUS, NUMBER_AFTER_INITIAL_ZERO, NUMBER_AFTER_INITIAL_NON_ZERO, NUMBER_AFTER_FULL_STOP, NUMBER_AFTER_DECIMAL, NUMBER_AFTER_E, NUMBER_AFTER_E_AND_SIGN, NUMBER_AFTER_E_AND_DIGIT, SEPARATOR, BOM_OR_START, BOM, } function TokenizerStateToString(tokenizerState: TokenizerStates): string { return [ "START", "ENDED", "ERROR", "TRUE1", "TRUE2", "TRUE3", "FALSE1", "FALSE2", "FALSE3", "FALSE4", "NULL1", "NULL2", "NULL3", "STRING_DEFAULT", "STRING_AFTER_BACKSLASH", "STRING_UNICODE_DIGIT_1", "STRING_UNICODE_DIGIT_2", "STRING_UNICODE_DIGIT_3", "STRING_UNICODE_DIGIT_4", "STRING_INCOMPLETE_CHAR", "NUMBER_AFTER_INITIAL_MINUS", "NUMBER_AFTER_INITIAL_ZERO", "NUMBER_AFTER_INITIAL_NON_ZERO", "NUMBER_AFTER_FULL_STOP", "NUMBER_AFTER_DECIMAL", "NUMBER_AFTER_E", "NUMBER_AFTER_E_AND_SIGN", "NUMBER_AFTER_E_AND_DIGIT", "SEPARATOR", "BOM_OR_START", "BOM", ][tokenizerState]; } /** The options that a {@linkcode Tokenizer} can be created with. */ export interface TokenizerOptions { /** * The size, in bytes, of the buffer to accumulate strings into. Defaults to * `0`, which accumulates them as JavaScript strings instead of buffering. * Values between 1 and 4 are also treated as no buffering. A reasonable size * when buffering is `64 * 1024` (64 KB). See {@linkcode BufferedString}. */ stringBufferSize?: number; /** * The size, in bytes, of the buffer to accumulate numbers into. Defaults to * `0`, which accumulates them as JavaScript strings instead of buffering. */ numberBufferSize?: number; /** * The separator between consecutive JSON documents in the stream, for example * `"\n"` for newline-delimited JSON. Defaults to `undefined`, which ends the * tokenizer after the first document. Set it to `""` to accept documents that * follow each other with no delimiter at all. */ separator?: string; /** * Whether to emit a token for the part of a string or number tokenized so far * every time a chunk ends in the middle of one. Defaults to `false`. Partial * tokens are flagged with `partial: true`. */ emitPartialTokens?: boolean; } const defaultOpts: TokenizerOptions = { stringBufferSize: 0, numberBufferSize: 0, separator: undefined, emitPartialTokens: false, }; /** The error thrown when the tokenizer is misconfigured or hits invalid JSON. */ export class TokenizerError extends Error { /** * @param message What went wrong. */ constructor(message: string) { super(message); // Typescript is broken. This is a workaround Object.setPrototypeOf(this, TokenizerError.prototype); } } // A non-integer buffer size (e.g. 0.5) silently truncates when passed to // `new Uint8Array(size)` (0.5 becomes a *zero-length* buffer) instead of // throwing, so every appended byte gets silently dropped rather than // buffered -- corrupting the parsed value instead of failing loudly. function validateBufferSize(name: string, size: number | undefined): void { if (size === undefined) return; if (!Number.isInteger(size) || size < 0) { throw new TokenizerError( `Invalid "${name}": ${size}. Expected a non-negative integer.`, ); } } // Byte length of the UTF-8 character starting with `leadByte`. Invalid or // continuation lead bytes fall through to 3/4 here and are rejected later by // the fatal TextDecoder when the bytes are actually decoded. function utf8SequenceLength(leadByte: number): number { if (leadByte >= 194 && leadByte <= 223) return 2; if (leadByte <= 239) return 3; return 4; } // Index just past the last COMPLETE multi-byte character of the run starting at // `start`. Stops at the first ASCII byte, or at a character whose bytes would // run past the end of the buffer (a boundary split the caller carries over). function multiByteRunEnd(buffer: Uint8Array, start: number): number { let j = start; while (j < buffer.length && buffer[j] >= 128) { const seqLength = utf8SequenceLength(buffer[j]); if (j + seqLength > buffer.length) break; // split across the chunk boundary j += seqLength; } return j; } /** * A JSON-compliant tokenizer that turns a utf-8 stream into JSON tokens. * * Data is pushed in with {@linkcode Tokenizer.write} and the resulting tokens * come back through the {@linkcode Tokenizer.onToken} callback, which the user * is expected to override. Feed the tokens to a `TokenParser` to get JSON * values back, or use a `JSONParser`, which chains both. * * @example * ```ts * import Tokenizer from "@streamparser/json/tokenizer.js"; * * const tokenizer = new Tokenizer({ separator: "\n" }); * tokenizer.onToken = ({ token, value, offset }) => { * // process the token * }; * tokenizer.onError = (err) => console.error(err); * * tokenizer.write('{ "test": ["a"] }'); * tokenizer.end(); * ``` */ export default class Tokenizer { private state = TokenizerStates.BOM_OR_START; private bom?: number[]; private bomIndex = 0; private emitPartialTokens: boolean; private separator?: string; private separatorBytes?: Uint8Array; private separatorIndex = 0; private escapedCharsByteLength = 0; private bufferedString: StringBuilder; private bufferedNumber: StringBuilder; private unicode?: string; // unicode escapes private highSurrogate?: number; private bytes_remaining = 0; // number of bytes remaining in multi byte utf8 char to read after split boundary private bytes_in_sequence = 0; // bytes in multi byte utf8 char to read private char_split_buffer = new Uint8Array(4); // for rebuilding chars split before boundary is reached // A raw (non-escaped) surrogate pair split across two separate string // `write()` calls: a lone trailing high surrogate held back from the end // of a string chunk, to be prepended to the next string chunk before // encoding. Encoding each chunk independently would otherwise replace the // unpaired surrogate with U+FFFD, since TextEncoder always produces // well-formed UTF-8 and lone surrogates have no valid UTF-8 encoding. private pendingStringSurrogate?: string; private encoder = new TextEncoder(); private offset = -1; private streamByteLength = 0; // Total bytes consumed across all write() calls before the current one /** * @param opts How to tokenize. See {@linkcode TokenizerOptions}. */ constructor(opts?: TokenizerOptions) { opts = { ...defaultOpts, ...opts }; validateBufferSize("stringBufferSize", opts.stringBufferSize); validateBufferSize("numberBufferSize", opts.numberBufferSize); this.emitPartialTokens = opts.emitPartialTokens === true; this.bufferedString = opts.stringBufferSize && opts.stringBufferSize > 4 ? new BufferedString(opts.stringBufferSize) : new NonBufferedString(); this.bufferedNumber = opts.numberBufferSize && opts.numberBufferSize > 0 ? new BufferedString(opts.numberBufferSize) : new NonBufferedString(); this.separator = opts.separator; this.separatorBytes = opts.separator ? this.encoder.encode(opts.separator) : undefined; } /** Whether the tokenizer is ended, and thus no longer accepting data. */ public get isEnded(): boolean { return this.state === TokenizerStates.ENDED; } // Appends the code unit decoded from one \uXXXX escape, matching // JSON.parse's handling of surrogates: a valid high/low surrogate pair // combines into one character; an unpaired high or low surrogate is kept // as a raw UTF-16 code unit rather than replaced or dropped (JS strings // are free to contain lone surrogates; only encoding them as UTF-8 bytes // is lossy, which is why appendCharCode -- not the encoder -- is used for // them). private appendUnicodeCodeUnit(intVal: number): void { if (this.highSurrogate !== undefined) { if (intVal >= 0xdc00 && intVal <= 0xdfff) { // <56320,57343> - valid low surrogate: combine with the pending // high surrogate into a single character. const unicodeString = String.fromCharCode(this.highSurrogate, intVal); const unicodeBuffer = this.encoder.encode(unicodeString); this.bufferedString.appendBuf(unicodeBuffer); // len(\u0000)=6 minus the fact you're appending len(buf) this.escapedCharsByteLength += 6 - unicodeBuffer.byteLength; this.highSurrogate = undefined; return; } // Not a matching low surrogate: the pending high surrogate stands on // its own, and intVal is processed independently below. this.flushPendingHighSurrogate(); } if (intVal >= 0xd800 && intVal <= 0xdbff) { // <55296,56319> - high surrogate: defer until we know whether a // matching low surrogate follows. this.highSurrogate = intVal; this.escapedCharsByteLength += 6; return; } if (intVal >= 0xdc00 && intVal <= 0xdfff) { // <56320,57343> - lone low surrogate with no preceding high // surrogate: keep as a raw code unit. this.bufferedString.appendCharCode(intVal); this.escapedCharsByteLength += 6; return; } const unicodeString = String.fromCharCode(intVal); const unicodeBuffer = this.encoder.encode(unicodeString); this.bufferedString.appendBuf(unicodeBuffer); // len(\u0000)=6 minus the fact you're appending len(buf) this.escapedCharsByteLength += 6 - unicodeBuffer.byteLength; } private flushPendingHighSurrogate(): void { if (this.highSurrogate !== undefined) { this.bufferedString.appendCharCode(this.highSurrogate); this.highSurrogate = undefined; } } // Stash the leading bytes of a multi-byte character split across the chunk // boundary; STRING_INCOMPLETE_CHAR completes it from the next chunk. private startIncompleteChar(buffer: Uint8Array, start: number): void { this.bytes_in_sequence = utf8SequenceLength(buffer[start]); this.bytes_remaining = start + this.bytes_in_sequence - buffer.length; this.char_split_buffer.set(buffer.subarray(start)); this.state = TokenizerStates.STRING_INCOMPLETE_CHAR; } /** * Pushes the next chunk of the JSON stream into the tokenizer. * * Tokenizing happens synchronously, so every token in `input` is emitted * through {@linkcode Tokenizer.onToken} before this returns. A chunk may end * anywhere, including in the middle of a multi-byte character; the rest of it * is picked up from the next chunk. * * @param input The chunk to tokenize: a string, a `TypedArray`, or any * iterable of utf-8 byte values. * @throws {TokenizerError} If the data is not valid JSON and no * {@linkcode Tokenizer.onError} callback has been set. */ public write(input: Iterable | string): void { try { let buffer: Uint8Array; if (input instanceof Uint8Array) { buffer = input; } else if (typeof input === "string") { if (this.pendingStringSurrogate !== undefined) { input = this.pendingStringSurrogate + input; this.pendingStringSurrogate = undefined; } const lastCharCode = input.charCodeAt(input.length - 1); if (lastCharCode >= 0xd800 && lastCharCode <= 0xdbff) { // Lone high surrogate at the very end of this chunk: hold it back // instead of encoding it (and corrupting it into U+FFFD) alone, // in case the next chunk supplies its matching low surrogate. this.pendingStringSurrogate = input[input.length - 1]; input = input.slice(0, -1); } buffer = this.encoder.encode(input); } else if (ArrayBuffer.isView(input)) { buffer = new Uint8Array( input.buffer, input.byteOffset, input.byteLength, ); } else if ( input !== null && typeof input === "object" && typeof (input as Iterable)[Symbol.iterator] === "function" ) { // Any Iterable, not just literal Arrays (e.g. Set, Map // values(), a generator) -- matching the public write() signature, // which already types `input` as Iterable | string. buffer = Uint8Array.from(input as Iterable); } else { throw new TypeError( "Unexpected type. The `write` function only accepts Iterables (e.g. Arrays, Sets, Generators), TypedArrays and Strings.", ); } for (let i = 0; i < buffer.length; i += 1) { const n = buffer[i]; // get current byte from buffer switch (this.state) { // @ts-expect-error fall through case case TokenizerStates.BOM_OR_START: if (n === 0xef) { this.bom = [0xef, 0xbb, 0xbf]; this.bomIndex += 1; this.state = TokenizerStates.BOM; continue; } if (input instanceof Uint16Array) { if (n === 0xfe) { this.bom = [0xfe, 0xff]; this.bomIndex += 1; this.state = TokenizerStates.BOM; continue; } if (n === 0xff) { this.bom = [0xff, 0xfe]; this.bomIndex += 1; this.state = TokenizerStates.BOM; continue; } } if (input instanceof Uint32Array) { if (n === 0x00) { this.bom = [0x00, 0x00, 0xfe, 0xff]; this.bomIndex += 1; this.state = TokenizerStates.BOM; continue; } if (n === 0xff) { this.bom = [0xff, 0xfe, 0x00, 0x00]; this.bomIndex += 1; this.state = TokenizerStates.BOM; continue; } } case TokenizerStates.START: this.offset += 1; if (this.separatorBytes && n === this.separatorBytes[0]) { if (this.separatorBytes.length === 1) { this.state = TokenizerStates.START; this.onToken({ token: TokenType.SEPARATOR, value: this.separator as string, offset: this.offset + this.separatorBytes.length - 1, }); continue; } this.state = TokenizerStates.SEPARATOR; continue; } if ( n === charset.SPACE || n === charset.NEWLINE || n === charset.CARRIAGE_RETURN || n === charset.TAB ) { // whitespace continue; } if (n === charset.LEFT_CURLY_BRACKET) { this.onToken({ token: TokenType.LEFT_BRACE, value: "{", offset: this.offset, }); continue; } if (n === charset.RIGHT_CURLY_BRACKET) { this.onToken({ token: TokenType.RIGHT_BRACE, value: "}", offset: this.offset, }); continue; } if (n === charset.LEFT_SQUARE_BRACKET) { this.onToken({ token: TokenType.LEFT_BRACKET, value: "[", offset: this.offset, }); continue; } if (n === charset.RIGHT_SQUARE_BRACKET) { this.onToken({ token: TokenType.RIGHT_BRACKET, value: "]", offset: this.offset, }); continue; } if (n === charset.COLON) { this.onToken({ token: TokenType.COLON, value: ":", offset: this.offset, }); continue; } if (n === charset.COMMA) { this.onToken({ token: TokenType.COMMA, value: ",", offset: this.offset, }); continue; } if (n === charset.LATIN_SMALL_LETTER_T) { this.state = TokenizerStates.TRUE1; continue; } if (n === charset.LATIN_SMALL_LETTER_F) { this.state = TokenizerStates.FALSE1; continue; } if (n === charset.LATIN_SMALL_LETTER_N) { this.state = TokenizerStates.NULL1; continue; } if (n === charset.QUOTATION_MARK) { this.bufferedString.reset(); this.escapedCharsByteLength = 0; this.state = TokenizerStates.STRING_DEFAULT; continue; } if (n >= charset.DIGIT_ONE && n <= charset.DIGIT_NINE) { this.bufferedNumber.reset(); this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_INITIAL_NON_ZERO; continue; } if (n === charset.DIGIT_ZERO) { this.bufferedNumber.reset(); this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_INITIAL_ZERO; continue; } if (n === charset.HYPHEN_MINUS) { this.bufferedNumber.reset(); this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_INITIAL_MINUS; continue; } break; // STRING case TokenizerStates.STRING_DEFAULT: if (n === charset.QUOTATION_MARK) { this.flushPendingHighSurrogate(); const string = this.bufferedString.toString(); this.state = TokenizerStates.START; this.onToken({ token: TokenType.STRING, value: string, offset: this.offset, }); this.offset += this.escapedCharsByteLength + this.bufferedString.byteLength + 1; continue; } if (n === charset.REVERSE_SOLIDUS) { this.state = TokenizerStates.STRING_AFTER_BACKSLASH; continue; } if (n >= 128) { this.flushPendingHighSurrogate(); // Decode the whole run of complete multi-byte characters in one // TextDecoder call, rather than one character at a time -- much // faster for multi-byte text (CJK/emoji). ASCII stays on the // per-character appendChar path below. const runEnd = multiByteRunEnd(buffer, i); if (runEnd > i) { this.bufferedString.appendBuf(buffer, i, runEnd); i = runEnd - 1; // the for-loop's i += 1 lands on runEnd } // A character straddling the chunk boundary is carried over to // the next chunk via STRING_INCOMPLETE_CHAR. if (runEnd < buffer.length && buffer[runEnd] >= 128) { this.startIncompleteChar(buffer, runEnd); i = buffer.length - 1; } continue; } if (n >= charset.SPACE) { this.flushPendingHighSurrogate(); let j = i; while (j < buffer.length) { const b = buffer[j]; if ( b < charset.SPACE || b >= 128 || b === charset.QUOTATION_MARK || b === charset.REVERSE_SOLIDUS ) break; j += 1; } // appendBuf is one TextDecoder call: worth it only once the run is // long enough to amortize that fixed cost. Short strings (keys, // ids) dominate real JSON, so append those char-by-char instead -- // always-appendBuf regresses key/record-heavy JSON ~12%. if (j - i >= 16) { this.bufferedString.appendBuf(buffer, i, j); } else { for (let k = i; k < j; k += 1) this.bufferedString.appendChar(buffer[k]); } i = j - 1; continue; } break; case TokenizerStates.STRING_INCOMPLETE_CHAR: { // check for carry over of a multi byte char split between data chunks // & fill temp buffer it with start of this data chunk up to the boundary limit set in the last iteration // The rest of the sequence might still not be complete if this chunk is smaller // than the number of bytes still missing (e.g. one byte at a time), so only // consume what's actually available and keep waiting otherwise. const available = Math.min(this.bytes_remaining, buffer.length - i); this.char_split_buffer.set( buffer.subarray(i, i + available), this.bytes_in_sequence - this.bytes_remaining, ); this.bytes_remaining -= available; if (this.bytes_remaining > 0) { i = buffer.length - 1; continue; } this.bufferedString.appendBuf( this.char_split_buffer, 0, this.bytes_in_sequence, ); i += available - 1; this.state = TokenizerStates.STRING_DEFAULT; continue; } case TokenizerStates.STRING_AFTER_BACKSLASH: { const controlChar = escapedSequences[n]; if (controlChar) { this.flushPendingHighSurrogate(); this.bufferedString.appendChar(controlChar); this.escapedCharsByteLength += 1; // len(\")=2 minus the fact you're appending len(controlChar)=1 this.state = TokenizerStates.STRING_DEFAULT; continue; } if (n === charset.LATIN_SMALL_LETTER_U) { this.unicode = ""; this.state = TokenizerStates.STRING_UNICODE_DIGIT_1; continue; } break; } case TokenizerStates.STRING_UNICODE_DIGIT_1: case TokenizerStates.STRING_UNICODE_DIGIT_2: case TokenizerStates.STRING_UNICODE_DIGIT_3: if ( (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) || (n >= charset.LATIN_CAPITAL_LETTER_A && n <= charset.LATIN_CAPITAL_LETTER_F) || (n >= charset.LATIN_SMALL_LETTER_A && n <= charset.LATIN_SMALL_LETTER_F) ) { this.unicode += String.fromCharCode(n); this.state += 1; continue; } break; case TokenizerStates.STRING_UNICODE_DIGIT_4: if ( (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) || (n >= charset.LATIN_CAPITAL_LETTER_A && n <= charset.LATIN_CAPITAL_LETTER_F) || (n >= charset.LATIN_SMALL_LETTER_A && n <= charset.LATIN_SMALL_LETTER_F) ) { const intVal = parseInt( this.unicode + String.fromCharCode(n), 16, ); this.appendUnicodeCodeUnit(intVal); this.state = TokenizerStates.STRING_DEFAULT; continue; } break; // Number case TokenizerStates.NUMBER_AFTER_INITIAL_MINUS: if (n === charset.DIGIT_ZERO) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_INITIAL_ZERO; continue; } if (n >= charset.DIGIT_ONE && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_INITIAL_NON_ZERO; continue; } break; case TokenizerStates.NUMBER_AFTER_INITIAL_ZERO: if (n === charset.FULL_STOP) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_FULL_STOP; continue; } if ( n === charset.LATIN_SMALL_LETTER_E || n === charset.LATIN_CAPITAL_LETTER_E ) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_E; continue; } i -= 1; this.state = TokenizerStates.START; this.emitNumber(); continue; case TokenizerStates.NUMBER_AFTER_INITIAL_NON_ZERO: if (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); continue; } if (n === charset.FULL_STOP) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_FULL_STOP; continue; } if ( n === charset.LATIN_SMALL_LETTER_E || n === charset.LATIN_CAPITAL_LETTER_E ) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_E; continue; } i -= 1; this.state = TokenizerStates.START; this.emitNumber(); continue; case TokenizerStates.NUMBER_AFTER_FULL_STOP: if (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_DECIMAL; continue; } break; case TokenizerStates.NUMBER_AFTER_DECIMAL: if (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); continue; } if ( n === charset.LATIN_SMALL_LETTER_E || n === charset.LATIN_CAPITAL_LETTER_E ) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_E; continue; } i -= 1; this.state = TokenizerStates.START; this.emitNumber(); continue; // @ts-expect-error fall through case case TokenizerStates.NUMBER_AFTER_E: if (n === charset.PLUS_SIGN || n === charset.HYPHEN_MINUS) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_E_AND_SIGN; continue; } case TokenizerStates.NUMBER_AFTER_E_AND_SIGN: if (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); this.state = TokenizerStates.NUMBER_AFTER_E_AND_DIGIT; continue; } break; case TokenizerStates.NUMBER_AFTER_E_AND_DIGIT: if (n >= charset.DIGIT_ZERO && n <= charset.DIGIT_NINE) { this.bufferedNumber.appendChar(n); continue; } i -= 1; this.state = TokenizerStates.START; this.emitNumber(); continue; // TRUE case TokenizerStates.TRUE1: if (n === charset.LATIN_SMALL_LETTER_R) { this.state = TokenizerStates.TRUE2; continue; } break; case TokenizerStates.TRUE2: if (n === charset.LATIN_SMALL_LETTER_U) { this.state = TokenizerStates.TRUE3; continue; } break; case TokenizerStates.TRUE3: if (n === charset.LATIN_SMALL_LETTER_E) { this.state = TokenizerStates.START; this.onToken({ token: TokenType.TRUE, value: true, offset: this.offset, }); this.offset += 3; continue; } break; // FALSE case TokenizerStates.FALSE1: if (n === charset.LATIN_SMALL_LETTER_A) { this.state = TokenizerStates.FALSE2; continue; } break; case TokenizerStates.FALSE2: if (n === charset.LATIN_SMALL_LETTER_L) { this.state = TokenizerStates.FALSE3; continue; } break; case TokenizerStates.FALSE3: if (n === charset.LATIN_SMALL_LETTER_S) { this.state = TokenizerStates.FALSE4; continue; } break; case TokenizerStates.FALSE4: if (n === charset.LATIN_SMALL_LETTER_E) { this.state = TokenizerStates.START; this.onToken({ token: TokenType.FALSE, value: false, offset: this.offset, }); this.offset += 4; continue; } break; // NULL case TokenizerStates.NULL1: if (n === charset.LATIN_SMALL_LETTER_U) { this.state = TokenizerStates.NULL2; continue; } break; case TokenizerStates.NULL2: if (n === charset.LATIN_SMALL_LETTER_L) { this.state = TokenizerStates.NULL3; continue; } break; case TokenizerStates.NULL3: if (n === charset.LATIN_SMALL_LETTER_L) { this.state = TokenizerStates.START; this.onToken({ token: TokenType.NULL, value: null, offset: this.offset, }); this.offset += 3; continue; } break; case TokenizerStates.SEPARATOR: this.separatorIndex += 1; if ( !this.separatorBytes || n !== this.separatorBytes[this.separatorIndex] ) { break; } if (this.separatorIndex === this.separatorBytes.length - 1) { this.state = TokenizerStates.START; this.onToken({ token: TokenType.SEPARATOR, value: this.separator as string, offset: this.offset + this.separatorIndex, }); this.separatorIndex = 0; } continue; // BOM support case TokenizerStates.BOM: if (n === this.bom![this.bomIndex]) { if (this.bomIndex === this.bom!.length - 1) { this.state = TokenizerStates.START; this.bom = undefined; this.bomIndex = 0; continue; } this.bomIndex += 1; continue; } break; case TokenizerStates.ENDED: if ( n === charset.SPACE || n === charset.NEWLINE || n === charset.CARRIAGE_RETURN || n === charset.TAB ) { // whitespace continue; } } throw new TokenizerError( `Unexpected "${String.fromCharCode( n, )}" at chunk position "${i}" (absolute position "${ this.streamByteLength + i }") in state ${TokenizerStateToString(this.state)}`, ); } this.streamByteLength += buffer.length; if (this.emitPartialTokens) { switch (this.state) { case TokenizerStates.TRUE1: case TokenizerStates.TRUE2: case TokenizerStates.TRUE3: this.onToken({ token: TokenType.TRUE, value: true, offset: this.offset, partial: true, }); break; case TokenizerStates.FALSE1: case TokenizerStates.FALSE2: case TokenizerStates.FALSE3: case TokenizerStates.FALSE4: this.onToken({ token: TokenType.FALSE, value: false, offset: this.offset, partial: true, }); break; case TokenizerStates.NULL1: case TokenizerStates.NULL2: case TokenizerStates.NULL3: this.onToken({ token: TokenType.NULL, value: null, offset: this.offset, partial: true, }); break; case TokenizerStates.STRING_DEFAULT: { const string = this.bufferedString.toString(); this.onToken({ token: TokenType.STRING, value: string, offset: this.offset, partial: true, }); break; } case TokenizerStates.NUMBER_AFTER_INITIAL_ZERO: case TokenizerStates.NUMBER_AFTER_INITIAL_NON_ZERO: case TokenizerStates.NUMBER_AFTER_DECIMAL: case TokenizerStates.NUMBER_AFTER_E_AND_DIGIT: try { this.onToken({ token: TokenType.NUMBER, value: this.parseNumber(this.bufferedNumber.toString()), offset: this.offset, partial: true, }); } catch { // Number couldn't be parsed. Do nothing. } } } } catch (err: unknown) { this.error(err as Error); } } private emitNumber(): void { this.onToken({ token: TokenType.NUMBER, value: this.parseNumber(this.bufferedNumber.toString()), offset: this.offset, }); this.offset += this.bufferedNumber.byteLength - 1; } /** * Turns the characters of a JSON number into a JavaScript value. * * Equivalent to `Number(numberStr)`. Override it to handle numbers that a * JavaScript number can't represent, for example by keeping them as strings. * * @param numberStr The number, as it appeared in the JSON stream. * @returns The parsed number. */ protected parseNumber(numberStr: string): number { return Number(numberStr); } /** * Puts the tokenizer in an error state and reports `err` through * {@linkcode Tokenizer.onError}. The tokenizer can't be used afterwards. * * @param err What went wrong. */ public error(err: Error): void { if (this.state !== TokenizerStates.ENDED) { this.state = TokenizerStates.ERROR; } this.onError(err); } /** * Signals that the stream is over, flushing any number that was still being * tokenized and then ending the tokenizer, which can't be used afterwards. * * @throws {TokenizerError} If the stream ended in the middle of a token and no * {@linkcode Tokenizer.onError} callback has been set. */ public end(): void { switch (this.state) { case TokenizerStates.NUMBER_AFTER_INITIAL_ZERO: case TokenizerStates.NUMBER_AFTER_INITIAL_NON_ZERO: case TokenizerStates.NUMBER_AFTER_DECIMAL: case TokenizerStates.NUMBER_AFTER_E_AND_DIGIT: this.state = TokenizerStates.ENDED; this.emitNumber(); this.onEnd(); break; case TokenizerStates.BOM_OR_START: case TokenizerStates.START: case TokenizerStates.ERROR: this.state = TokenizerStates.ENDED; this.onEnd(); break; default: this.error( new TokenizerError( `Tokenizer ended in the middle of a token (state: ${TokenizerStateToString( this.state, )}). Either not all the data was received or the data was invalid.`, ), ); } } /** * Called with every token found in the stream. Override it to consume them; * by default it throws. * * @param parsedToken The token and where it was found. */ // biome-ignore lint/correctness/noUnusedFunctionParameters: override point; the parameter is part of the public signature public onToken(parsedToken: ParsedTokenInfo): void { // Override me throw new TokenizerError( 'Can\'t emit tokens before the "onToken" callback has been set up.', ); } /** * Called when the data can't be tokenized. Override it to handle errors * asynchronously; by default it throws, so the error surfaces out of the * {@linkcode Tokenizer.write} or {@linkcode Tokenizer.end} call that caused it. * * @param err What went wrong. */ public onError(err: Error): void { // Override me throw err; } /** Called once the tokenizer has ended. Override it to react to that; by default it does nothing. */ public onEnd(): void { // Override me } }