export type Tag = "code" | "url" | "mention" | "emoji" | "custom-emoji" | "word" | "unknown";
export interface Token {
    text: string;
    tag: Tag;
}
export interface TokenMatcher {
    /** What kind of token is being matched */
    tag: Tag;
    /** Regex pattern to search for this token */
    regex: RegExp;
    /** Optionally applies a transformation to the matched text */
    transform?: (match: string) => string;
}
/**
 * Splits the input string using one TokenMatcher into a list of strings and tokens where:
 * - strings are the parts of the input that do not match the matcher (trimmed)
 * - tokens are the parts of the input that do match the matcher (@see Token)
 *
 * For example, if the matcher matches emojis:
 * > "hello world" -> ["hello world"]
 * > "😃" -> [{ ..."😃" }]
 * > "notemoji 😃 notemoji" -> ["notemoji", { ..."😃" }, "notemoji"]
 */
export declare const splitByToken: (input: string, matcher: TokenMatcher) => (string | Token)[];
/**
 * Tokenizes a string recursively.
 * It is assumed that all matchers with index `< matcherIndex` have already been
 * tried and failed to match, so we are clear to test for matchers `>= matcherIndex`.
 *
 * @param matcherIndex the index of the matcher to use in the Matchers array
 */
export declare const tokenizeStep: (input: string, matcherIndex: number) => Token[];
/** Tokenizes a string into a list of tokens */
export declare const tokenize: (input: string) => Token[];