/// import Stream from 'stream'; import { SentenceExample } from './parsers'; export declare const SPLIT_STRATEGIES: { id: (id: string) => string; 'raw-sentence': (id: string, sentence: string) => string; sentence: (id: string, sentence: string, program: string) => string; spotify: (id: string, sentence: string, program: string) => string; aggregate: (id: string, sentence: string, program: string) => string; program: (id: string, sentence: string, program: string) => string; combination: (id: string, sentence: string, program: string) => string; 'context-and-program': (id: string, sentence: string, program: string, context?: string | undefined) => string; }; interface DatasetSplitterOptions { locale: string; rng: () => number; evalProbability: number; forDevices?: string[]; evalOnSynthetic: boolean; useEvalFlag: boolean; splitStrategy?: keyof typeof SPLIT_STRATEGIES; train: Stream.Writable; eval: Stream.Writable; test?: Stream.Writable; } export default class DatasetSplitter extends Stream.Writable { private _rng; private _evalProbability; private _forDevices; private _evalOnSynthetic; private _useEvalFlag; private _train; private _eval; private _test; private _splitStrategy; private _dedupedevtestMakeKey; private _devtestset; private _dedupeddevtestset; private _trainset; constructor(options: DatasetSplitterOptions); _final(callback: () => void): void; /** Check if this example can potentially be used for evaluation. If this method returns true, the example is a candidate for sampling in the evaluation sets (validation/test). Synthetic (S) and augmented (P) sentences are excluded if `evalOnSynthetic` is false (default), If `useEvalFlag` was passed as option, this method considers the `eval` flag (E flag in TSV format), otherwise all other sentences are potentially included in the evaluation sets. */ private _isFlaggedForEval; private _doWriteLower; private _handleOne; private _handleMany; _write(row: SentenceExample, encoding: BufferEncoding, callback: (err?: Error) => void): void; } export {};