/** * Dataset cleaner (Dataset-uploader.md §2.3 — the `dataset_clean` tool's core). * * Loads a sharegpt dataset JSONL, scrubs EVERY string field of every sample * (the JSON tree is walked; structure is never changed), applies optional * quality filters, and writes `.clean.jsonl` next to the input. * * - minTurns — drop samples with fewer assistant ("gpt") turns * - maxSampleChars — drop samples longer than this many chars * - keepOnlyToolCalls — drop samples without any function_call entry * * The scrubber receives `extraValues` collected by the caller (secret-store * values, backend/.env.local values). This module never reads any store and * never prints values — only per-kind redaction counts. */ export interface CleanDatasetOptions { /** Output path. Default: `.clean.jsonl` next to the input. */ outputPath?: string; /** Minimum number of assistant ("gpt") turns per sample. */ minTurns?: number; /** Maximum total characters per sample (conversation values + system). */ maxSampleChars?: number; /** When true, drop samples that contain no function_call entries. */ keepOnlyToolCalls?: boolean; /** * Opaque secret values to redact literally (secret-store values, .env * values). The cleaner never reads any store itself. */ extraValues?: string[]; } export interface CleanDatasetResult { inputPath: string; outputPath: string; total: number; kept: number; removed: number; malformedLines: number; removedByFilter: { minTurns: number; maxSampleChars: number; keepOnlyToolCalls: number; }; /** Per-kind redaction counts (only kinds with count > 0, stable order). */ redactions: { kind: string; count: number; }[]; } /** Derive the default clean output path: `export-x.jsonl` → `export-x.clean.jsonl`. */ export declare function defaultCleanPath(inputPath: string): string; /** * Clean a dataset JSONL file: scrub all string fields, apply optional * filters, write `.clean.jsonl` next to the input. * * Throws when the input file cannot be read. Malformed lines are counted, * not fatal. */ export declare function cleanDataset(inputPath: string, options?: CleanDatasetOptions): CleanDatasetResult; //# sourceMappingURL=cleaner.d.ts.map