/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest. * ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and * re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one * label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's * overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a * set rather than one — chaining eight single-slice overlays would leave seven dead directories and * an eight-deep base chain for what is one version). * * Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is * read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON. * * Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs * --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-slice.ts * --input --output --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet * --input --output /train/ node * scripts/assemble-overlay-manifest.ts --base /MANIFEST.json --new-dir \ * --modal-root /data/corpus/versioned// --version \ * --slice-parquet --source --note "..." * * # then push the overlay to R2 + sync + `modal run -d ... --config .yaml --resume none`. */ type ManifestSlice = Record & { path: string; source?: string; }; /** * The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every * corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact * — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the * word is banned in this tree and the ratchet's baseline is zero. */ declare const LEGACY_SLICES_KEY: "shards"; declare const LEGACY_ROWS_PER_SLICE_KEY: "rows_per_shard"; interface BaseManifest { corpus_version?: string; schema: unknown; rows_per_slice?: unknown; row_group_size: unknown; slices?: ManifestSlice[]; counts: { train: number; val: number; test: number; }; total_rows: number; /** * The pre-rename spellings, present on every corpus built before 2026-09-01. */ [LEGACY_SLICES_KEY]?: ManifestSlice[]; [LEGACY_ROWS_PER_SLICE_KEY]?: unknown; } /** * A base manifest's slice list under either key. * * The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer * measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the * loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all. */ export declare function baseManifestSlices(base: BaseManifest): ManifestSlice[]; /** * One parquet to add, with the source label its rows carry. */ export interface OverlaySlice { parquet: string; source: string; } export interface OverlayManifestOptions { base: string; newDir: string; modalRoot: string; version: string; /** * The parquets this overlay adds, in the order they should appear after the base's own. */ slices: readonly OverlaySlice[]; note: string; } /** * Resolve a base manifest's slice path to the mounted corpus tree used by Modal. */ export declare function rerootBaseSlicePath(path: string, baseManifestPath: string): string; export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise; export {}; //# sourceMappingURL=overlay-manifest.d.ts.map