/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Build per-script parquet files from the DeepSeek-generated transliteration jsonl and emit the * corpus-v0.4.0 manifest that combines them with the existing kryptonite + v0.3.0 files. * * This module sits beside `kryptonite.ts`. Both modules use the same composition pattern: take a * base manifest, append new parquet files, write a combined manifest. Differences specific to * transliteration: * * - One jsonl contains rows from N target scripts (source = `deepseek-translit-`). We bucket by * `source` and write one parquet file per script so `audit.ts` can attribute each file to its * synthetic source without relying on filename-prefix inference. * - Each file is written to `train/part-translit-.parquet` (distinct from kryptonite's * `part-0000.parquet`, which v0.4.0's first builder already produced). * - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 file paths are * rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined manifest so * all paths share one container-friendly form. * * See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for prompts, model and the * reproducibility interface. * * Invoke via `mailwoman corpus slice translit \ * --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \ * --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/manifest.json \ * --out-dir /data/corpus/versioned/v0.4.0` */ import { type PathBuilderLike } from "path-ts"; export interface TranslitOverlayOptions { jsonl: string; baseManifest: string; outDir: PathBuilderLike; /** * Default `"0.4.0"`. */ corpusVersion?: string; /** * Default `"/data/"`. */ canonicalPathPrefix?: string; /** * Prefix currently used in the base manifest's file paths, to be rewritten * to {@link TranslitOverlayOptions.canonicalPathPrefix}. * * Defaults to `dataRootPath()` with a trailing slash — the root that wrote those paths. * Pass it explicitly when translating a manifest generated under a different * `$MAILWOMAN_DATA_ROOT` than the one you are running with. */ legacyPathPrefix?: string; } export declare function buildTranslitOverlay(options: TranslitOverlayOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=translit.d.ts.map