/**
 * @copyright Sister Software
 * @license AGPL-3.0
 * @author Teffen Ellis, et al.
 *
 *   Final output sliceer for the corpus pipeline.
 *
 *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
 *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
 *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
 *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
 *
 *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
 *   which is the standard ML-corpus default (and PyArrow's default). Documented in
 *   `DECISIONS.md`.
 *
 *   Layout under `<outputDir>`:
 *
 *   ```
 *   corpus-v<version>/
 *   MANIFEST.json
 *   train/
 *     part-0000.parquet
 *     part-0001.parquet
 *     ...
 *   val/
 *     part-0000.parquet
 *   test/
 *     part-0000.parquet
 * ```
 *
 *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
 *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
 *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
 *   close — cheap relative to writing it).
 */
import { type PathBuilderLike } from "path-ts";
import type { LabeledRow } from "#types";
import type { SplitName } from "#utils/split";
/**
 * Row groups are written at this cadence within a slice.
 */
export declare const ROW_GROUP_SIZE = 50000;
/**
 * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
 */
export declare function escapeSQLString(value: string): string;
/**
 * An open DuckDB connection, re-exported so consumers of {@link connectDuckDB} can name the type without their own
 * static dependency on the optional peer.
 */
export type { DuckDBConnection } from "@duckdb/node-api";
/**
 * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
 * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
 */
export declare function connectDuckDB(): Promise<import("@duckdb/node-api").DuckDBConnection>;
/**
 * Snappy is the codec selected for corpus slices.
 */
export declare const SLICE_COMPRESSION: "SNAPPY";
export interface ParquetFieldDefinition {
    type: "UTF8" | "INT32";
    compression: typeof SLICE_COMPRESSION;
    repeated?: boolean;
    optional?: boolean;
}
export type ParquetSchemaDefinition<T> = Record<Extract<keyof T, string>, ParquetFieldDefinition>;
/**
 * A single Parquet row shape. The index signature allows callers to carry source fields before projection.
 *
 * Optional fields are represented as null in the Arrow table and read back as null.
 */
export interface ParquetRow {
    raw: string;
    tokens: readonly string[];
    labels: readonly string[];
    span_starts: readonly number[];
    span_ends: readonly number[];
    span_tags: readonly string[];
    country: string;
    locale?: string | null;
    source: string;
    source_id: string;
    corpus_version: string;
    license: string;
    synth_method?: string | null;
    synth_base_id?: string | null;
    [key: string]: unknown;
}
/**
 * Column names emitted into every slice. Matches `ParquetRow`.
 */
export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
/**
 * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
 * tokens/labels arrays. Compression is per-column SNAPPY.
 */
export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
export declare function writeParquetRows(rows: readonly ParquetRow[], path: string): Promise<void>;
/**
 * Stream rows from a local Parquet file in DuckDB-managed chunks.
 *
 * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
 * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
 */
export declare function streamParquetRows<T>(path: string, columns?: readonly string[], options?: {
    limit?: number;
}): AsyncGenerator<T>;
/**
 * Per-slice metadata captured in `MANIFEST.json`.
 */
export interface SliceDescriptor {
    split: SplitName;
    path: string;
    format: "parquet";
    compression: typeof SLICE_COMPRESSION;
    rows: number;
    bytes: number;
    sha256: string;
    first_source_id: string;
    last_source_id: string;
    /**
     * The slice's corpus source slug, when the writer knows it. `audit.ts` prefers this over inferring the source from
     * `first_source_id`'s prefix; `writeSlices` itself writes multi-source slices and leaves it unset.
     */
    source?: string;
}
export interface SliceManifest {
    corpus_version: string;
    schema: readonly string[];
    rows_per_slice: number;
    row_group_size: number;
    slices: SliceDescriptor[];
    counts: Record<SplitName, number>;
    total_rows: number;
}
export interface WriteSlicesOptions {
    /**
     * Root output directory; corpus version dir is created beneath.
     */
    outputDir: PathBuilderLike;
    /**
     * Corpus version stamped onto rows + into the output directory name.
     */
    corpusVersion: string;
    /**
     * Max rows per `.parquet` slice. Default 1_000_000 per the Phase 1 plan.
     */
    rowsPerSlice?: number;
}
/**
 * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
 * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
 * lookup table.
 *
 * Splits with no rows can be omitted (or passed as an empty iterable); `writeSlices` skips them.
 */
export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
/**
 * Project a labeled row to the Parquet schema.
 *
 * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
 * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
 * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
 */
export declare function rowToParquet(row: LabeledRow): ParquetRow;
/**
 * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
 * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
 * Parquet file from disk.
 *
 * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
 * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
 * for the new wire-up.
 */
export declare function writeSlices(perSplit: PerSplitRows, opts: WriteSlicesOptions): Promise<SliceManifest>;
//# sourceMappingURL=parquet.d.ts.map