/* tslint:disable */ /* eslint-disable */ /** * Read a Parquet file into Arrow data. * * This returns an Arrow table in WebAssembly memory. To transfer the Arrow table to JavaScript * memory you have two options: * * - (Easier): Call {@linkcode Table.intoIPCStream} to construct a buffer that can be parsed with * Arrow JS's `tableFromIPC` function. * - (More performant but bleeding edge): Call {@linkcode Table.intoFFI} to construct a data * representation that can be parsed zero-copy from WebAssembly with * [arrow-js-ffi](https://github.com/kylebarron/arrow-js-ffi) using `parseTable`. * * Example with IPC stream: * * ```js * import { tableFromIPC } from "apache-arrow"; * import initWasm, {readParquet} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmTable = readParquet(parquetUint8Array); * const arrowTable = tableFromIPC(arrowWasmTable.intoIPCStream()); * ``` * * Example with `arrow-js-ffi`: * * ```js * import { parseTable } from "arrow-js-ffi"; * import initWasm, {readParquet, wasmMemory} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * const WASM_MEMORY = wasmMemory(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmTable = readParquet(parquetUint8Array); * const ffiTable = arrowWasmTable.intoFFI(); * const arrowTable = parseTable( * WASM_MEMORY.buffer, * ffiTable.arrayAddrs(), * ffiTable.schemaAddr() * ); * ``` * * @param parquet_file Uint8Array containing Parquet data * @param options * * Options for reading Parquet data. Optional keys include: * * - `batchSize`: The number of rows in each batch. If not provided, the upstream parquet * default is 1024. * - `rowGroups`: Only read data from the provided row group indexes. * - `limit`: Provide a limit to the number of rows to be read. * - `offset`: Provide an offset to skip over the given number of rows. * - `columns`: The column names from the file to read. */ export function readParquet(parquet_file: Uint8Array, options?: ReaderOptions | null): Table; /** * Read an Arrow schema from a Parquet file in memory. * * This returns an Arrow schema in WebAssembly memory. To transfer the Arrow schema to JavaScript * memory you have two options: * * - (Easier): Call {@linkcode Schema.intoIPCStream} to construct a buffer that can be parsed with * Arrow JS's `tableFromIPC` function. This results in an Arrow JS Table with zero rows but a * valid schema. * - (More performant but bleeding edge): Call {@linkcode Schema.intoFFI} to construct a data * representation that can be parsed zero-copy from WebAssembly with * [arrow-js-ffi](https://github.com/kylebarron/arrow-js-ffi) using `parseSchema`. * * Example with IPC Stream: * * ```js * import { tableFromIPC } from "apache-arrow"; * import initWasm, {readSchema} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmSchema = readSchema(parquetUint8Array); * const arrowTable = tableFromIPC(arrowWasmSchema.intoIPCStream()); * const arrowSchema = arrowTable.schema; * ``` * * Example with `arrow-js-ffi`: * * ```js * import { parseSchema } from "arrow-js-ffi"; * import initWasm, {readSchema, wasmMemory} from "parquet-wasm"; * * // Instantiate the WebAssembly context * await initWasm(); * const WASM_MEMORY = wasmMemory(); * * const resp = await fetch("https://example.com/file.parquet"); * const parquetUint8Array = new Uint8Array(await resp.arrayBuffer()); * const arrowWasmSchema = readSchema(parquetUint8Array); * const ffiSchema = arrowWasmSchema.intoFFI(); * const arrowTable = parseSchema(WASM_MEMORY.buffer, ffiSchema.addr()); * const arrowSchema = arrowTable.schema; * ``` * * @param parquet_file Uint8Array containing Parquet data */ export function readSchema(parquet_file: Uint8Array): Schema; /** * Returns a handle to this wasm instance's `WebAssembly.Memory` */ export function wasmMemory(): Memory; /** * Returns a handle to this wasm instance's `WebAssembly.Table` which is the indirect function * table used by Rust */ export function _functionTable(): FunctionTable; /** * Supported compression algorithms. * * Codecs added in format version X.Y can be read by readers based on X.Y and later. * Codec support may vary between readers based on the format version and * libraries available at runtime. */ export enum Compression { UNCOMPRESSED = 0, SNAPPY = 1, GZIP = 2, BROTLI = 3, /** * @deprecated as of Parquet 2.9.0. * Switch to LZ4_RAW */ LZ4 = 4, ZSTD = 5, LZ4_RAW = 6, LZO = 7, } /** * Encodings supported by Parquet. * Not all encodings are valid for all types. These enums are also used to specify the * encoding of definition and repetition levels. */ export enum Encoding { /** * Default byte encoding. * - BOOLEAN - 1 bit per value, 0 is false; 1 is true. * - INT32 - 4 bytes per value, stored as little-endian. * - INT64 - 8 bytes per value, stored as little-endian. * - FLOAT - 4 bytes per value, stored as little-endian. * - DOUBLE - 8 bytes per value, stored as little-endian. * - BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes. * - FIXED_LEN_BYTE_ARRAY - just the bytes are stored. */ PLAIN = 0, /** * **Deprecated** dictionary encoding. * * The values in the dictionary are encoded using PLAIN encoding. * Since it is deprecated, RLE_DICTIONARY encoding is used for a data page, and * PLAIN encoding is used for dictionary page. */ PLAIN_DICTIONARY = 1, /** * Group packed run length encoding. * * Usable for definition/repetition levels encoding and boolean values. */ RLE = 2, /** * Bit packed encoding. * * This can only be used if the data has a known max width. * Usable for definition/repetition levels encoding. */ BIT_PACKED = 3, /** * Delta encoding for integers, either INT32 or INT64. * * Works best on sorted data. */ DELTA_BINARY_PACKED = 4, /** * Encoding for byte arrays to separate the length values and the data. * * The lengths are encoded using DELTA_BINARY_PACKED encoding. */ DELTA_LENGTH_BYTE_ARRAY = 5, /** * Incremental encoding for byte arrays. * * Prefix lengths are encoded using DELTA_BINARY_PACKED encoding. * Suffixes are stored using DELTA_LENGTH_BYTE_ARRAY encoding. */ DELTA_BYTE_ARRAY = 6, /** * Dictionary encoding. * * The ids are encoded using the RLE encoding. */ RLE_DICTIONARY = 7, /** * Encoding for floating-point data. * * K byte-streams are created where K is the size in bytes of the data type. * The individual bytes of an FP value are scattered to the corresponding stream and * the streams are concatenated. * This itself does not reduce the size of the data but can lead to better compression * afterwards. */ BYTE_STREAM_SPLIT = 8, } /** * The Parquet version to use when writing */ export enum WriterVersion { V1 = 0, V2 = 1, } export type ReaderOptions = { /* The number of rows in each batch. If not provided, the upstream parquet default is 1024. */ batchSize?: number; /* Only read data from the provided row group indexes. */ rowGroups?: number[]; /* Provide a limit to the number of rows to be read. */ limit?: number; /* Provide an offset to skip over the given number of rows. */ offset?: number; /* The column names from the file to read. */ columns?: string[]; /* The number of concurrent requests to make in the async reader. */ concurrency?: number; }; export type FunctionTable = WebAssembly.Table; export type Memory = WebAssembly.Memory; export type SchemaMetadata = Map; /** * Metadata for a Parquet column chunk. */ export class ColumnChunkMetaData { private constructor(); free(): void; /** * File where the column chunk is stored. * * If not set, assumed to belong to the same file as the metadata. * This path is relative to the current file. */ filePath(): string | undefined; /** * Byte offset in `file_path()`. */ fileOffset(): bigint; /** * Path (or identifier) of this column. */ columnPath(): string[]; /** * All encodings used for this column. */ encodings(): any[]; /** * Total number of values in this column chunk. */ numValues(): number; /** * Compression for this column. */ compression(): Compression; /** * Returns the total compressed data size of this column chunk. */ compressedSize(): number; /** * Returns the total uncompressed data size of this column chunk. */ uncompressedSize(): number; } /** * An Arrow array exported to FFI. * * Using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi), you can view or copy Arrow * these objects to JavaScript. * * Note that this also includes an ArrowSchema C struct as well, so that extension type * information can be maintained. * ## Memory management * * Note that this array will not be released automatically. You need to manually call `.free()` to * release memory. */ export class FFIData { private constructor(); free(): void; /** * Access the pointer to the * [`ArrowArray`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `RecordBatch` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` */ arrayAddr(): number; /** * Access the pointer to the * [`ArrowSchema`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `Field` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` */ schemaAddr(): number; } export class FFISchema { private constructor(); free(): void; /** * Access the pointer to the * [`ArrowSchema`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct. This can be viewed or copied (without serialization) to an Arrow JS `Field` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmRecordBatch: FFIRecordBatch = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmRecordBatch.arrayAddr(), * wasmRecordBatch.schemaAddr(), * true * ); * ``` */ addr(): number; } /** * A representation of an Arrow C Stream in WebAssembly memory exposed as FFI-compatible * structs through the Arrow C Data Interface. * * Unlike other Arrow implementations outside of JS, this always stores the "stream" fully * materialized as a sequence of Arrow chunks. */ export class FFIStream { private constructor(); free(): void; /** * Get the total number of elements in this stream */ numArrays(): number; /** * Get the pointer to the ArrowSchema FFI struct */ schemaAddr(): number; /** * Get the pointer to one ArrowArray FFI struct for a given chunk index and column index * * Access the pointer to one * [`ArrowArray`](https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions) * struct representing one of the internal `RecordBatch`es. This can be viewed or copied (without serialization) to an Arrow JS `RecordBatch` by * using [`arrow-js-ffi`](https://github.com/kylebarron/arrow-js-ffi). You can access the * [`WebAssembly.Memory`](https://developer.mozilla.org/en-US/docs/WebAssembly/JavaScript_interface/Memory) * instance by using {@linkcode wasmMemory}. * * **Example**: * * ```ts * import * as arrow from "apache-arrow"; * import { parseRecordBatch } from "arrow-js-ffi"; * * const wasmTable: FFITable = ... * const wasmMemory: WebAssembly.Memory = wasmMemory(); * * const jsBatches: arrow.RecordBatch[] = [] * for (let i = 0; i < wasmTable.numBatches(); i++) { * // Pass `true` to copy arrays across the boundary instead of creating views. * const jsRecordBatch = parseRecordBatch( * wasmMemory.buffer, * wasmTable.arrayAddr(i), * wasmTable.schemaAddr(), * true * ); * jsBatches.push(jsRecordBatch); * } * const jsTable = new arrow.Table(jsBatches); * ``` * * @param chunk number The chunk index to use * @returns number pointer to an ArrowArray FFI struct in Wasm memory */ arrayAddr(chunk: number): number; arrayAddrs(): Uint32Array; drop(): void; } /** * Metadata for a Parquet file. */ export class FileMetaData { private constructor(); free(): void; /** * Returns version of this file. */ version(): number; /** * Returns number of rows in the file. */ numRows(): number; /** * String message for application that wrote this file. * * This should have the following format: * ` version (build )`. * * ```shell * parquet-mr version 1.8.0 (build 0fda28af84b9746396014ad6a415b90592a98b3b) * ``` */ createdBy(): string | undefined; /** * Returns key_value_metadata of this file. */ keyValueMetadata(): Map; } /** * Global Parquet metadata. */ export class ParquetMetaData { private constructor(); free(): void; /** * Returns file metadata as reference. */ fileMetadata(): FileMetaData; /** * Returns number of row groups in this file. */ numRowGroups(): number; /** * Returns row group metadata for `i`th position. * Position should be less than number of row groups `num_row_groups`. */ rowGroup(i: number): RowGroupMetaData; /** * Returns row group metadata for all row groups */ rowGroups(): RowGroupMetaData[]; } /** * A group of columns of equal length in WebAssembly memory with an associated {@linkcode Schema}. */ export class RecordBatch { private constructor(); free(): void; /** * Export this RecordBatch to FFI structs according to the Arrow C Data Interface. * * This method **does not consume** the RecordBatch, so you must remember to call {@linkcode * RecordBatch.free} to release the resources. The underlying arrays are reference counted, so * this method does not copy data, it only prevents the data from being released. */ toFFI(): FFIData; /** * Export this RecordBatch to FFI structs according to the Arrow C Data Interface. * * This method **does consume** the RecordBatch, so the original RecordBatch will be * inaccessible after this call. You must still call {@linkcode FFIRecordBatch.free} after * you've finished using the FFIRecordBatch. */ intoFFI(): FFIData; /** * Consume this RecordBatch and convert to an Arrow IPC Stream buffer */ intoIPCStream(): Uint8Array; /** * Override the schema of this [`RecordBatch`] * * Returns an error if `schema` is not a superset of the current schema * as determined by [`Schema::contains`] */ withSchema(schema: Schema): RecordBatch; /** * Return a new RecordBatch where each column is sliced * according to `offset` and `length` */ slice(offset: number, length: number): RecordBatch; /** * Returns the total number of bytes of memory occupied physically by this batch. */ getArrayMemorySize(): number; /** * The number of rows in this RecordBatch. */ readonly numRows: number; /** * The number of columns in this RecordBatch. */ readonly numColumns: number; /** * The {@linkcode Schema} of this RecordBatch. */ readonly schema: Schema; } /** * Metadata for a Parquet row group. */ export class RowGroupMetaData { private constructor(); free(): void; /** * Number of columns in this row group. */ numColumns(): number; /** * Returns column chunk metadata for `i`th column. */ column(i: number): ColumnChunkMetaData; /** * Returns column chunk metadata for all columns */ columns(): ColumnChunkMetaData[]; /** * Number of rows in this row group. */ numRows(): number; /** * Total byte size of all uncompressed column data in this row group. */ totalByteSize(): number; /** * Total size of all compressed column data in this row group. */ compressedSize(): number; } /** * A named collection of types that defines the column names and types in a RecordBatch or Table * data structure. * * A Schema can also contain extra user-defined metadata either at the Table or Column level. * Column-level metadata is often used to define [extension * types](https://arrow.apache.org/docs/format/Columnar.html#extension-types). */ export class Schema { private constructor(); free(): void; /** * Export this schema to an FFISchema object, which can be read with arrow-js-ffi. * * This method **does not consume** the Schema, so you must remember to call {@linkcode * Schema.free} to release the resources. The underlying arrays are reference counted, so * this method does not copy data, it only prevents the data from being released. */ toFFI(): FFISchema; /** * Export this Table to FFI structs according to the Arrow C Data Interface. * * This method **does consume** the Table, so the original Table will be * inaccessible after this call. You must still call {@linkcode FFITable.free} after * you've finished using the FFITable. */ intoFFI(): FFISchema; /** * Consume this schema and convert to an Arrow IPC Stream buffer */ intoIPCStream(): Uint8Array; /** * Sets the metadata of this `Schema` to be `metadata` and returns a new object */ withMetadata(metadata: SchemaMetadata): Schema; /** * Find the index of the column with the given name. */ indexOf(name: string): number; /** * Returns an immutable reference to the Map of custom metadata key-value pairs. */ metadata(): SchemaMetadata; } /** * A Table in WebAssembly memory conforming to the Apache Arrow spec. * * A Table consists of one or more {@linkcode RecordBatch} objects plus a {@linkcode Schema} that * each RecordBatch conforms to. */ export class Table { private constructor(); free(): void; /** * Access a RecordBatch from the Table by index. * * @param index The positional index of the RecordBatch to retrieve. * @returns a RecordBatch or `null` if out of range. */ recordBatch(index: number): RecordBatch | undefined; recordBatches(): RecordBatch[]; /** * Export this Table to FFI structs according to the Arrow C Data Interface. * * This method **does not consume** the Table, so you must remember to call {@linkcode * Table.free} to release the resources. The underlying arrays are reference counted, so * this method does not copy data, it only prevents the data from being released. */ toFFI(): FFIStream; /** * Export this Table to FFI structs according to the Arrow C Data Interface. * * This method **does consume** the Table, so the original Table will be * inaccessible after this call. You must still call {@linkcode FFITable.free} after * you've finished using the FFITable. */ intoFFI(): FFIStream; /** * Consume this table and convert to an Arrow IPC Stream buffer */ intoIPCStream(): Uint8Array; /** * Create a table from an Arrow IPC Stream buffer */ static fromIPCStream(buf: Uint8Array): Table; /** * Returns the total number of bytes of memory occupied physically by all batches in this * table. */ getArrayMemorySize(): number; /** * Access the Table's {@linkcode Schema}. */ readonly schema: Schema; /** * The number of batches in the Table */ readonly numBatches: number; }