import { TextField, Token, RawToken, RawTokenColumn } from "../../../types";
/**
 * Tokenize a document, but allowing for multiple text fields to be concatenated as different fields.
 * @param {*} textFields  An array of objects, where each object has the structure {name, value}. 'name' becomes the name of the field, 'value' is the text
 *                         each item can also have an 'offset' key with an integer value, in case the value is a subset starting at the [offset] character (this is needed to get the correct positions in the original document)
 *                         each item can also have a 'unit_start' and 'unit_end' key, each with an integer value to indicate where in this textField the codingUnit starts/ends.
 *                         If both unit_start and unit_end is omitted, the whole text is considered codingUnit.
 *                         As an alternative to unit_start and unit_end, can also have context_before and context_after to specify context, which should both be strings
 * @returns
 */
export declare const parseTokens: (textFields: TextField[]) => Token[];
export declare const importTokens: (tokens: RawToken[] | RawTokenColumn) => Token[];
/**
 * changes tokens in column format
 *  {{offset: [1,2], token: ["hello","world"]}
 * to row format
 *  [{offset: 1, token: "hello"}, {offset: 2, token: "world"}]
 *
 * row format is easier to work with, but column format is more efficient
 * so allow it to be used as input.
 * @param {} tokens
 */
export declare const tokensColumnToRow: (tokens: RawTokenColumn) => RawToken[];