/**
 * Code-aware tokenizer.
 *
 * Default path: pure local simple code tokenizer (zero heavy deps).
 * Optional path: StarCoder2 tokenizer via optional `@huggingface/transformers`.
 */
export interface CodeToken {
    readonly text: string;
    readonly id: number;
}
export interface TokenizerOptions {
    readonly modelPath?: string;
    readonly cacheDir?: string;
    /** Force simple tokenizer even if transformers is installed. */
    readonly preferSimple?: boolean;
}
/** Pure local tokenizer: camelCase / snake / identifiers — no network, no ML wheels. */
export declare function simpleCodeTokenize(code: string): string[];
/**
 * Code tokenizer with optional StarCoder2 backend.
 * Falls back to simple local tokenization when HF is absent or fails.
 */
export declare class CodeTokenizer {
    private tokenizer;
    private initialized;
    private initPromise;
    private modelPath;
    private preferSimple;
    private backend;
    constructor(options?: TokenizerOptions);
    backendName(): 'simple' | 'starcoder2';
    initialize(): Promise<void>;
    private doInitialize;
    tokenize(code: string): Promise<string[]>;
    extractTerms(code: string): Promise<Map<string, number>>;
    isReady(): boolean;
}
export declare function getTokenizer(): CodeTokenizer;
export declare function tokenize(code: string): Promise<string[]>;
export declare function extractTerms(code: string): Promise<Map<string, number>>;
export declare function initializeTokenizer(): Promise<void>;
//# sourceMappingURL=code-tokenizer.d.ts.map