import type { ColumnChunk, ColumnIndex, CompressionCodec, DecodedArray, Encoding, KeyValue, OffsetIndex, SchemaElement, Statistics } from 'hyparquet' export type Compressor = (input: Uint8Array) => Uint8Array export type Compressors = { [K in CompressionCodec]?: Compressor } // Superset of parquet types with automatic conversions export type BasicType = 'BOOLEAN' | 'INT32' | 'INT64' | 'FLOAT' | 'DOUBLE' | 'BYTE_ARRAY' | 'STRING' | 'JSON' | 'TIMESTAMP' | 'UUID' | 'FLOAT16' | 'GEOMETRY' | 'GEOGRAPHY' | 'VARIANT' | 'FILE' // Recursive variant shredding config: a scalar BasicType, an array of one // element shred type ([elem] = array-of-elem), or an object mapping field names // to shred types. The single-element array is a shape template; only index 0 is read. export type ShredType = BasicType | ShredType[] | { [field: string]: ShredType } export interface ParquetWriteOptions { writer: Writer columnData: ColumnSource[] schema?: SchemaElement[] codec?: CompressionCodec // global default codec, default 'SNAPPY' compressors?: Compressors // custom compressors, required for any codec other than SNAPPY and UNCOMPRESSED statistics?: boolean // enable column statistics, default true statisticsTruncateLength?: number // max bytes of a STRING or BYTE_ARRAY min/max in statistics and column indexes (default 16, Infinity keeps whole values; FIXED_LEN_BYTE_ARRAY, JSON, BSON and DECIMAL bounds are never truncated) rowGroupSize?: number | number[] // number of rows per row group pageSize?: number // target uncompressed page size in bytes, default 1048576 dictionarySize?: number // hard cap on distinct-value bytes per column chunk dictionary; unset = keep any dictionary that at least halves the encoded size kvMetadata?: KeyValue[] createdBy?: string // FileMetaData created_by, default 'hyparquet' } // Same write options as ParquetWriteOptions, but the data is supplied as rows // plus a column spec instead of pre-transposed columnData. export interface ParquetWriteRowsOptions extends Omit { rows: Record[] | Iterable> | AsyncIterable> columns: Omit[] } export interface ColumnSource { name: string data: DecodedArray type?: BasicType nullable?: boolean encoding?: Encoding codec?: CompressionCodec // per-column codec override, default ParquetWriteOptions.codec statistics?: boolean // per-column statistics override (also encoding_stats and geospatial statistics), default ParquetWriteOptions.statistics columnIndex?: boolean // write column indexes, default false offsetIndex?: boolean // write offset indexes, default true shredding?: true | ShredType // variant shredding config (true = auto-detect) bloomFilter?: boolean | BloomFilterOptions // write bloom filter, default false } export interface PageData { values: DecodedArray definitionLevels: number[] repetitionLevels: number[] maxDefinitionLevel: number } export interface BloomFilterOptions { fpp?: number // false positive probability, default 0.01 maxBytes?: number // cap the filter at the largest power of two <= maxBytes, default 1 MiB } export interface ColumnEncoder { columnName: string element: SchemaElement schemaPath: SchemaElement[] codec: CompressionCodec compressors: Compressors stats: boolean statisticsTruncateLength?: number pageSize: number dictionarySize?: number // Spec: If ColumnIndex is present, OffsetIndex must also be present columnIndex: boolean offsetIndex: boolean encoding?: Encoding // user-specified encoding bloomFilter?: boolean | BloomFilterOptions } // nan_count and nan_counts are written but not yet typed by hyparquet export interface ChunkStatistics extends Statistics { nan_count?: bigint } export interface PageColumnIndex extends ColumnIndex { nan_counts?: bigint[] } export interface PageIndexes { chunk: ColumnChunk columnIndex?: PageColumnIndex offsetIndex?: OffsetIndex bloomFilter?: Uint32Array // finalized SBBF blocks } export interface Writer { buffer: ArrayBuffer view: DataView offset: number // total bytes written ensure(size: number): void flush?(): void | Promise finish(): void | Promise getBuffer(): ArrayBuffer getBytes(): Uint8Array appendUint8(value: number): void appendUint32(value: number): void appendInt32(value: number): void appendInt64(value: bigint): void appendFloat32(value: number): void appendFloat64(value: number): void appendBuffer(buffer: ArrayBuffer): void appendBytes(value: Uint8Array): void appendVarInt(value: number): void appendVarBigInt(value: bigint): void appendZigZag(value: number | bigint): void } export type ThriftObject = { [ key: `field_${number}` ]: ThriftType } export type ThriftField = [fieldId: number, type: number, value: ThriftType] export interface PreconvertedThriftStruct { fields: ThriftField[] } export type ThriftType = boolean | number | bigint | string | Uint8Array | PreconvertedThriftStruct | ThriftType[] | ThriftObject | undefined