/**
 * @copyright Sister Software
 * @license AGPL-3.0
 * @author Teffen Ellis, et al.
 *
 *   SentencePiece tokenizer wrapper over `@mailwoman/sentencepiece-wasm` (google/sentencepiece
 *   v0.2.2 with the native-offsets binding — task #26).
 *
 *   History: the previous runtime (`@sctg/sentencepiece-js`, an emscripten build of an OLDER
 *   sentencepiece whose binding never exposed the offset-carrying proto API) forced this file to
 *   RECONSTRUCT char offsets by re-walking the input string alongside the emitted pieces — ~90
 *   lines of cursor arithmetic with two documented hazard classes (byte-fallback desync, fixed by
 *   hand; surrogate-pair accounting, deferred) and one undocumented one (normalizer-changed
 *   surfaces: a piece like `DŽ` for input `DŽ` desyncs a literal-length cursor). SentencePiece
 *   itself has always known the answer: `Encode(text, &SentencePieceText)` yields per-piece
 *   `begin`/`end` BYTE offsets with the invariant `utf8(text).slice(begin, end) == surface` and
 *   contiguity between consecutive pieces — including the "zero-width except the last piece owns
 *   the character's span" behavior for byte-fallback runs that the old reconstruction implemented
 *   manually (verified byte-for-byte in the swap's parity battery, 1,066 fixture rows).
 *
 *   What this layer still owns:
 *
 *   - **Byte → UTF-16 conversion.** The native offsets are UTF-8 byte positions; the decoder wants
 *       JS string (UTF-16 code-unit) ranges. The conversion walks code points once per encode and
 *       is exact for non-BMP input (the old shim's deferred hazard, now covered by tests).
 *   - **Leading-whitespace trim.** A `▁`-prefixed piece's native span INCLUDES the whitespace the
 *       sentinel consumed (surface " Rock" for piece `▁Rock`); the decoder's contract has always
 *       been starts-at-the-word (`start` points at "R"). Trimming preserves the shipped decode
 *       byte-exactly, and collapses the bare-`▁` piece to the zero-width-after-space range the
 *       word grouper expects.
 *
 *   The wrapper supports two load modes:
 *
 *   - `loadFromBase64(b64)` — for tests and browser usage where the model arrives as bytes.
 *   - `loadFromFile(path)` — Node-only convenience (dynamic `node:fs` import keeps the browser
 *       bundle clean).
 */
/**
 * SentencePiece's word-boundary marker (U+2581 LOWER ONE EIGHTH BLOCK).
 */
export declare const SPACE_SENTINEL = "\u2581";
/**
 * A tokenized piece paired with its char-range in the original input.
 */
export interface TokenizedPiece {
    /**
     * The piece exactly as the tokenizer emitted it (with `▁` preserved where present).
     */
    piece: string;
    /**
     * The vocab id for this piece.
     */
    id: number;
    /**
     * Inclusive start char offset in the original input.
     */
    start: number;
    /**
     * Exclusive end char offset in the original input.
     */
    end: number;
}
export interface EncodeResult {
    pieces: TokenizedPiece[];
    ids: number[];
}
export declare class MailwomanTokenizer {
    private readonly processor;
    private readonly module;
    private constructor();
    private static loadFromBytes;
    /**
     * Load from a base64-encoded `tokenizer.model`. Use for in-memory / test / browser setups.
     */
    static loadFromBase64(b64: string): Promise<MailwomanTokenizer>;
    /**
     * Load from a path to a `tokenizer.model` file on disk. **Node-only** — the dynamic `node:fs` import keeps this
     * method out of the static dependency graph so the rest of the tokenizer bundles cleanly for the browser. Calling it
     * in a browser throws at runtime; use `loadFromBase64` (or the URL-fetching loaders in `@mailwoman/neural-web`)
     * instead.
     */
    static loadFromFile(modelPath: string): Promise<MailwomanTokenizer>;
    /**
     * Tokenize `text` to pieces + ids + native char offsets.
     *
     * The returned `pieces[i].piece` matches what the Python `sp.EncodeAsPieces(text)[i]` returns, and `pieces[i].id`
     * matches `sp.EncodeAsIDs(text)[i]`. Offsets come from SentencePiece's own `SentencePieceText` proto (byte
     * positions), converted to UTF-16 and whitespace-trimmed — see the file header for the two conventions this layer
     * owns.
     */
    encode(text: string): EncodeResult;
    /**
     * Decode a list of ids back to a string. Delegates to the underlying processor.
     */
    decode(ids: number[] | Int32Array): string;
}
//# sourceMappingURL=tokenizer.d.ts.map