/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema. * * Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through * PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT * `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT * PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact * logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST) for the string * arrays, and `INTEGER[]` (LIST) for the span offsets — in the exact column order below. * Verified field-for-field against the PyArrow original (same column order, same logical types, * same `list` child naming, same values), so a PyArrow reader sees an identical * table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`), * which is blind to physical layout. INT32 matches the corpus's native TS writer * (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay * rides alongside. * * Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source, * source_id, corpus_version, license, synth_method, synth_base_id. * * The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it * on every labeled row, so a row arriving without it came from a producer that hasn't migrated — * writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row * number, instead. * * Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output * /tmp/part-po-box.parquet */ /** * Options for {@linkcode jsonlToParquet}. */ export interface JSONLToParquetOptions { /** * The labeled-row JSONL to convert. */ input: string; /** * The parquet shard to write. */ output: string; /** * Parquet row-group size. Default 50000. */ rowGroupSize?: number; } /** * Summary returned by {@linkcode jsonlToParquet}. */ export interface JSONLToParquetSummary { read: number; written: number; outPath: string; } /** * Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */ export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=jsonl-to-parquet.d.ts.map