/**
 * @copyright Sister Software
 * @license AGPL-3.0
 * @author Teffen Ellis, et al.
 *
 *   Tokenizer interface for alignment.
 *
 *   Two implementations live in the corpus package:
 *
 *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
 *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
 *        default for in-container alignment tests and as a fallback when no SentencePiece model is
 *        available.
 *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
 *        trained on the corpus. Same interface, different splits. Locked against the corpus version
 *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
 *
 *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
 *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
 *   spans, independent of how the tokenizer chose its splits.
 */
/**
 * A token with its character span in the source string.
 */
export interface TokenSpan {
    /**
     * The token text, possibly normalized (case unchanged here; tokenizers may differ).
     */
    text: string;
    /**
     * Inclusive start offset (UTF-16 code-unit index) in the source string.
     */
    start: number;
    /**
     * Exclusive end offset in the source string. `text === source.slice(start, end)`.
     */
    end: number;
}
/**
 * A tokenizer that maps a string to a sequence of `TokenSpan`s.
 */
export interface Tokenizer {
    tokenize(text: string): readonly TokenSpan[];
}
/**
 * Whitespace + punctuation tokenizer (pure JS).
 *
 * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
 * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
 * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
 *
 * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
 * tokenizer will preserve all bytes via byte-fallback.
 */
export declare function whitespaceTokenizer(): Tokenizer;
//# sourceMappingURL=tokenize.d.ts.map