import { Grammar, SyntaxToken } from './types';
/**
 * Core tokenizer that converts source code into syntax tokens based on a grammar definition.
 * This is a local replacement for Prism tokenize.
 *
 * The tokenizer works by:
 * 1. Pre-processing greedy patterns to claim their matched regions first
 * 2. Iterating through all grammar rules in order
 * 3. Finding regex matches in the source text
 * 4. Splitting the text into matched (typed) and unmatched (plain) segments
 * 5. Recursively tokenizing nested content when an `inside` grammar is specified
 *
 * @param {string} code - The source code string to tokenize.
 * @param {Grammar} grammar - The grammar rules to apply for tokenization.
 * @returns {(string | SyntaxToken)[]} An array of plain strings and syntax tokens.
 */
export declare function tokenize(code: string, grammar: Grammar): (string | SyntaxToken)[];