import type { TokenPattern, TokenSpan } from "./types.ts";
/**
 * Scan text for token matches.
 *
 * Simple patterns use prefix-first scanning: find the prefix, greedily
 * consume body-alphabet chars, then try to split at prefix boundaries
 * where a valid token exists on the right-hand side.
 *
 * Structured patterns use fullRegex for initial matching, then try
 * truncating at prefix boundaries where a valid right-hand token exists.
 *
 * A truncation is only accepted when BOTH the left-side body validates
 * AND a real token match would be produced at the split point. This
 * prevents false splits when a variable-length body happens to contain
 * a prefix substring that doesn't lead to a valid token (too short,
 * too long, wrong characters, boundary-invalid, etc.).
 *
 * @param allPatterns - The full set of known patterns, used for boundary
 *   detection and right-hand-side validation. When using a types filter,
 *   this should include ALL patterns. Defaults to the provided patterns.
 */
export declare function scan(text: string, patterns: readonly TokenPattern[], allPatterns?: readonly TokenPattern[]): TokenSpan[];
/**
 * Shannon entropy in bits per character.
 */
export declare function shannonEntropy(s: string): number;