/**
 * A JSON-compliant tokenizer that turns a utf-8 stream into JSON tokens.
 *
 * @example
 * ```ts
 * import Tokenizer from "@streamparser/json/tokenizer.js";
 *
 * const tokenizer = new Tokenizer();
 * tokenizer.onToken = ({ token, value, offset }) => {
 *   // process the token
 * };
 *
 * tokenizer.write('{ "test": ["a"] }');
 * ```
 *
 * @module
 */
import type { ParsedTokenInfo } from "./utils/types/parsedTokenInfo.js";
/** The options that a {@linkcode Tokenizer} can be created with. */
export interface TokenizerOptions {
    /**
     * The size, in bytes, of the buffer to accumulate strings into. Defaults to
     * `0`, which accumulates them as JavaScript strings instead of buffering.
     * Values between 1 and 4 are also treated as no buffering. A reasonable size
     * when buffering is `64 * 1024` (64 KB). See {@linkcode BufferedString}.
     */
    stringBufferSize?: number;
    /**
     * The size, in bytes, of the buffer to accumulate numbers into. Defaults to
     * `0`, which accumulates them as JavaScript strings instead of buffering.
     */
    numberBufferSize?: number;
    /**
     * The separator between consecutive JSON documents in the stream, for example
     * `"\n"` for newline-delimited JSON. Defaults to `undefined`, which ends the
     * tokenizer after the first document. Set it to `""` to accept documents that
     * follow each other with no delimiter at all.
     */
    separator?: string;
    /**
     * Whether to emit a token for the part of a string or number tokenized so far
     * every time a chunk ends in the middle of one. Defaults to `false`. Partial
     * tokens are flagged with `partial: true`.
     */
    emitPartialTokens?: boolean;
}
/** The error thrown when the tokenizer is misconfigured or hits invalid JSON. */
export declare class TokenizerError extends Error {
    /**
     * @param message What went wrong.
     */
    constructor(message: string);
}
/**
 * A JSON-compliant tokenizer that turns a utf-8 stream into JSON tokens.
 *
 * Data is pushed in with {@linkcode Tokenizer.write} and the resulting tokens
 * come back through the {@linkcode Tokenizer.onToken} callback, which the user
 * is expected to override. Feed the tokens to a `TokenParser` to get JSON
 * values back, or use a `JSONParser`, which chains both.
 *
 * @example
 * ```ts
 * import Tokenizer from "@streamparser/json/tokenizer.js";
 *
 * const tokenizer = new Tokenizer({ separator: "\n" });
 * tokenizer.onToken = ({ token, value, offset }) => {
 *   // process the token
 * };
 * tokenizer.onError = (err) => console.error(err);
 *
 * tokenizer.write('{ "test": ["a"] }');
 * tokenizer.end();
 * ```
 */
export default class Tokenizer {
    private state;
    private bom?;
    private bomIndex;
    private emitPartialTokens;
    private separator?;
    private separatorBytes?;
    private separatorIndex;
    private escapedCharsByteLength;
    private bufferedString;
    private bufferedNumber;
    private unicode?;
    private highSurrogate?;
    private bytes_remaining;
    private bytes_in_sequence;
    private char_split_buffer;
    private pendingStringSurrogate?;
    private encoder;
    private offset;
    private streamByteLength;
    /**
     * @param opts How to tokenize. See {@linkcode TokenizerOptions}.
     */
    constructor(opts?: TokenizerOptions);
    /** Whether the tokenizer is ended, and thus no longer accepting data. */
    get isEnded(): boolean;
    private appendUnicodeCodeUnit;
    private flushPendingHighSurrogate;
    private startIncompleteChar;
    /**
     * Pushes the next chunk of the JSON stream into the tokenizer.
     *
     * Tokenizing happens synchronously, so every token in `input` is emitted
     * through {@linkcode Tokenizer.onToken} before this returns. A chunk may end
     * anywhere, including in the middle of a multi-byte character; the rest of it
     * is picked up from the next chunk.
     *
     * @param input The chunk to tokenize: a string, a `TypedArray`, or any
     * iterable of utf-8 byte values.
     * @throws {TokenizerError} If the data is not valid JSON and no
     * {@linkcode Tokenizer.onError} callback has been set.
     */
    write(input: Iterable<number> | string): void;
    private emitNumber;
    /**
     * Turns the characters of a JSON number into a JavaScript value.
     *
     * Equivalent to `Number(numberStr)`. Override it to handle numbers that a
     * JavaScript number can't represent, for example by keeping them as strings.
     *
     * @param numberStr The number, as it appeared in the JSON stream.
     * @returns The parsed number.
     */
    protected parseNumber(numberStr: string): number;
    /**
     * Puts the tokenizer in an error state and reports `err` through
     * {@linkcode Tokenizer.onError}. The tokenizer can't be used afterwards.
     *
     * @param err What went wrong.
     */
    error(err: Error): void;
    /**
     * Signals that the stream is over, flushing any number that was still being
     * tokenized and then ending the tokenizer, which can't be used afterwards.
     *
     * @throws {TokenizerError} If the stream ended in the middle of a token and no
     * {@linkcode Tokenizer.onError} callback has been set.
     */
    end(): void;
    /**
     * Called with every token found in the stream. Override it to consume them;
     * by default it throws.
     *
     * @param parsedToken The token and where it was found.
     */
    onToken(parsedToken: ParsedTokenInfo): void;
    /**
     * Called when the data can't be tokenized. Override it to handle errors
     * asynchronously; by default it throws, so the error surfaces out of the
     * {@linkcode Tokenizer.write} or {@linkcode Tokenizer.end} call that caused it.
     *
     * @param err What went wrong.
     */
    onError(err: Error): void;
    /** Called once the tokenizer has ended. Override it to react to that; by default it does nothing. */
    onEnd(): void;
}
//# sourceMappingURL=tokenizer.d.ts.map