/** * Assemble a Parquet file from typed columns. * * Layout: `PAR1`, one data page per column, the Thrift-encoded `FileMetaData` * footer, the footer length, `PAR1` again. v1 is deliberately plain — PLAIN * encoding, no compression, one row group — which is valid everywhere and keeps * the bytes deterministic, so every language port can produce the same file. * Spec: docs/specs/2026-07-19-typed-output-and-parquet-writer.md §6-7. */ import { type ColumnType, type OutputColumnType } from '../column-type.js'; import { type MapEntry } from './map.js'; import { type ColumnStatistics } from './statistics.js'; import type { TypedValue } from './convert.js'; /** A column's identity — everything the schema needs, without the data. */ export interface ParquetSchemaColumn { readonly name: string; readonly type: OutputColumnType; } /** * One cell. A scalar column holds a converted value; a LIST column holds the * row's raw element texts, and a MAP its raw pairs, because the definition * levels have to be decided (which entries are NULL) before anything is * converted. */ export type ParquetCell = TypedValue | readonly string[] | readonly MapEntry[]; export interface ParquetColumn extends ParquetSchemaColumn { /** One entry per ROW. */ readonly values: readonly ParquetCell[]; } export interface ChunkMeta { /** Where the chunk begins — the dictionary page when there is one. */ readonly offset: number; /** Where the DATA page begins, which is past the dictionary. */ readonly dataOffset: number; /** Set only when this chunk carries a dictionary page. */ readonly dictionaryOffset?: number; /** Bytes actually written — compressed. */ readonly totalSize: number; /** What those bytes would have been uncompressed, for the footer. */ readonly rawSize: number; /** Whichever codec actually saved bytes for this chunk. */ readonly codec: number; /** Level slots, NOT rows — for a list they differ. */ readonly numValues: number; readonly statistics: ColumnStatistics; /** * Which LEAF this chunk is, spelled as the reader addresses it. * * A scalar column is one leaf and one chunk; a list is one leaf under a * wrapper; a MAP is TWO — the key and the value — under one declared name. * The footer used to read the physical type and the path off the column, * which quietly assumed the first of those three. */ readonly path: readonly string[]; readonly physicalType: ColumnType; /** Whether the leaf carries level streams, which decides if RLE is declared. */ readonly hasLevels: boolean; } export interface GroupMeta { readonly chunks: readonly ChunkMeta[]; readonly numRows: number; } /** One row group's worth of values: `batch[columnIndex][row]`. */ export type ParquetBatch = readonly (readonly ParquetCell[])[]; /** * Emit a `.parquet` file as a sequence of chunks. Each batch becomes one ROW * GROUP, written and released before the next arrives, so peak memory is one * batch — not the whole dataset. Only the (small) per-group metadata is kept, * because the footer must be written last. */ /** * One row group, encoded and ready to place anywhere in a file. * * This is the unit that makes parallel writing possible: a group's BYTES do not * depend on where it sits, because page headers carry sizes and the only * offsets in the whole format live in the footer. So workers can each build * whole groups independently and a coordinator merely concatenates them and * fixes the offsets up at the end. * * `chunks` therefore carries offsets RELATIVE to the start of this block; the * caller shifts them once it knows where the block landed. */ export interface RowGroupBlock { readonly pages: Uint8Array[]; readonly chunks: ChunkMeta[]; readonly numRows: number; readonly byteLength: number; } /** Encode each batch into a position-independent row group. */ export declare function rowGroupBlocks(columns: readonly ParquetSchemaColumn[], batches: Iterable): Generator; /** Move a block's relative offsets to where the block actually landed. */ export declare function shiftChunks(chunks: readonly ChunkMeta[], by: number): ChunkMeta[]; /** * `column_orders` — the field that makes the statistics USABLE. * * The spec is explicit: a reader must ignore `min_value`/`max_value` unless * `FileMetaData.column_orders` says the sort order is TypeDefinedOrder. Without * it the bounds are there in the bytes and no conforming reader may act on * them, so every row group is decoded in full — which is exactly what the * statistics exist to avoid. The values we wrote were correct; nothing was * allowed to read them. * * One entry per LEAF column, in schema order — the same order the row groups * list their chunks in. A scalar or a list is one leaf; a MAP is TWO, its key * and its value, so this is not the count of declared columns. Getting that * wrong is not a subtle bug: readers refuse the file outright with "not enough * ColumnOrder values", which is at least loud. * * `ColumnOrder` is a union whose only member, `TYPE_ORDER`, holds an EMPTY * struct, so each entry is three bytes: the union's field header, the empty * struct's stop byte, and the union's own stop byte. */ /** How many leaf columns the declared ones expand to: a map is two, the rest one. */ export declare function leafCount(columns: readonly ParquetSchemaColumn[]): number; /** The footer: schema, row-group directory, and the trailing length + magic. */ export declare function parquetFooter(columns: readonly ParquetSchemaColumn[], groups: readonly GroupMeta[], numRows: number): Uint8Array; /** The four leading magic bytes; a file starts here. */ export declare const PARQUET_MAGIC: Uint8Array; export declare function writeParquetChunks(columns: readonly ParquetSchemaColumn[], batches: Iterable): Generator; /** Build a complete `.parquet` file in memory (one row group). */ export declare function writeParquet(columns: readonly ParquetColumn[], _numRows?: number): Uint8Array; //# sourceMappingURL=writer.d.ts.map