/**
 * @copyright Sister Software
 * @license AGPL-3.0
 * @author Teffen Ellis, et al.
 *
 *   Final output sharder for the corpus pipeline.
 *
 *   Phase 1 (#9) shipped JSONL shards + a Python (PyArrow) converter as the path to binary Parquet —
 *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
 *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
 *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
 *
 *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`, but `@dsnp/parquetjs` 1.7.0 only
 *   supports UNCOMPRESSED / GZIP / SNAPPY / BROTLI (see `node_modules/@dsnp/parquetjs/dist/lib/
 *   compression.js`). SNAPPY is the standard ML-corpus default (PyArrow's default too) and is the
 *   closest substitute on speed; revisit if @dsnp/parquetjs gains zstd support. Documented in
 *   `DECISIONS.md`.
 *
 *   Layout under `<outputDir>`:
 *
 *   ```
 *   corpus-v<version>/
 *   MANIFEST.json
 *   train/
 *     part-0000.parquet
 *     part-0001.parquet
 *     ...
 *   val/
 *     part-0000.parquet
 *   test/
 *     part-0000.parquet
 * ```
 *
 *   Each shard caps at `rowsPerShard` (default 1_000_000); within a shard, parquetjs flushes row
 *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
 *   shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
 *   close — cheap relative to writing it).
 */
import { type ParquetSchemaDefinition } from "@mailwoman/corpus/parquet-wrapper";
import type { LabeledRow } from "@mailwoman/corpus/types";
import type { SplitName } from "./split.ts";
/**
 * Row groups flush at this many rows (parquetjs internal cadence within a shard).
 */
export declare const ROW_GROUP_SIZE = 50000;
/**
 * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
 */
export declare const SHARD_COMPRESSION: "SNAPPY";
/**
 * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
 * `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on rows.
 */
export interface ParquetRow {
    raw: string;
    tokens: readonly string[];
    labels: readonly string[];
    span_starts: readonly number[];
    span_ends: readonly number[];
    span_tags: readonly string[];
    country: string;
    locale: string | null;
    source: string;
    source_id: string;
    corpus_version: string;
    license: string;
    synth_method: string | null;
    synth_base_id: string | null;
    [key: string]: unknown;
}
/**
 * Column names emitted into every shard. Matches `ParquetRow`.
 */
export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
/**
 * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
 * tokens/labels arrays. Compression is per-column SNAPPY.
 */
export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
/**
 * Per-shard metadata captured in `MANIFEST.json`.
 */
export interface ShardDescriptor {
    split: SplitName;
    path: string;
    format: "parquet";
    compression: typeof SHARD_COMPRESSION;
    rows: number;
    bytes: number;
    sha256: string;
    first_source_id: string;
    last_source_id: string;
}
export interface ShardManifest {
    corpus_version: string;
    schema: readonly string[];
    rows_per_shard: number;
    row_group_size: number;
    shards: ShardDescriptor[];
    counts: Record<SplitName, number>;
    total_rows: number;
}
export interface WriteShardsOptions {
    /**
     * Root output directory; corpus version dir is created beneath.
     */
    outputDir: string;
    /**
     * Corpus version stamped onto rows + into the output directory name.
     */
    corpusVersion: string;
    /**
     * Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan.
     */
    rowsPerShard?: number;
}
/**
 * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
 * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
 * lookup table.
 *
 * Splits with no rows can be omitted (or passed as an empty iterable); `writeShards` skips them.
 */
export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
/**
 * Project a labeled row to the Parquet schema.
 *
 * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
 * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the shard (the "builders
 * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
 */
export declare function rowToParquet(row: LabeledRow): ParquetRow;
/**
 * Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed sequentially so that
 * only one shard writer is open at a time — memory cost is bounded by the parquetjs row-group buffer (~`ROW_GROUP_SIZE
 * × row_size`), not by the labeled-row count.
 *
 * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
 * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
 * for the new wire-up.
 */
export declare function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptions): Promise<ShardManifest>;
//# sourceMappingURL=parquet.d.ts.map