import type { DataInfo, ParquetChunkMetadata, PartitionedDataInfoEntries, PColumn, PColumnValues, } from "@milaboratories/pl-model-common"; import { assertNever, dataInfoToEntries, isDataInfo, isDataInfoEntries, visitDataInfo, type BinaryChunk, type DataInfoEntries, type PColumnDataEntry, type PColumnKey, } from "@milaboratories/pl-model-common"; import { TreeNodeAccessor } from "../accessor"; import type { PColumnDataUniversal } from "../internal"; const PCD_PREFIX = "PColumnData/"; export const RT_RESOURCE_MAP = PCD_PREFIX + "ResourceMap"; export const RT_RESOURCE_MAP_PARTITIONED = PCD_PREFIX + "Partitioned/ResourceMap"; export const RT_JSON_PARTITIONED = PCD_PREFIX + "JsonPartitioned"; export const RT_BINARY_PARTITIONED = PCD_PREFIX + "BinaryPartitioned"; export const RT_PARQUET_PARTITIONED = PCD_PREFIX + "ParquetPartitioned"; const PCD_SUP_PREFIX = PCD_PREFIX + "Partitioned/"; export const RT_JSON_SUPER_PARTITIONED = PCD_SUP_PREFIX + "JsonPartitioned"; export const RT_BINARY_SUPER_PARTITIONED = PCD_SUP_PREFIX + "BinaryPartitioned"; export const RT_PARQUET_SUPER_PARTITIONED = PCD_SUP_PREFIX + "ParquetPartitioned"; export type PColumnResourceMapEntry = { key: PColumnKey; value: T; }; export type PColumnResourceMapData = { isComplete: boolean; data: PColumnResourceMapEntry[]; }; function populateResourceMapData( acc: TreeNodeAccessor | undefined, resourceParser: (acc: TreeNodeAccessor) => T | undefined, data: PColumnResourceMapEntry[], keyPrefix: PColumnKey = [], addEntriesWithNoData: boolean, ): boolean { if (acc === undefined) return false; switch (acc.resourceType.name) { case RT_RESOURCE_MAP: { let isComplete = acc.getInputsLocked(); for (const keyStr of acc.listInputFields()) { const value = acc.resolve({ field: keyStr, assertFieldType: "Input" }); const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey; const converted = value === undefined ? undefined : resourceParser(value); if (converted === undefined) isComplete = false; if (converted !== undefined || addEntriesWithNoData) data.push({ key, value: converted }); } return isComplete; } case RT_RESOURCE_MAP_PARTITIONED: { let isComplete = acc.getInputsLocked(); for (const keyStr of acc.listInputFields()) { const value = acc.resolve({ field: keyStr, assertFieldType: "Input" }); if (value === undefined) isComplete = false; else { const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey; const populateResult = populateResourceMapData( value, resourceParser, data, key, addEntriesWithNoData, ); isComplete = isComplete && populateResult; } } return isComplete; } default: throw new Error(`Unknown resource type: ${acc.resourceType.name}`); } } export function parseResourceMap( acc: TreeNodeAccessor | undefined, resourceParser: (acc: TreeNodeAccessor) => T | undefined, addEntriesWithNoData: false, ): PColumnResourceMapData>; export function parseResourceMap( acc: TreeNodeAccessor | undefined, resourceParser: (acc: TreeNodeAccessor) => T | undefined, addEntriesWithNoData: true, ): PColumnResourceMapData; export function parseResourceMap( acc: TreeNodeAccessor | undefined, resourceParser: (acc: TreeNodeAccessor) => T | undefined, addEntriesWithNoData: boolean = false, ): PColumnResourceMapData { const data: PColumnResourceMapEntry[] = []; const isComplete = populateResourceMapData(acc, resourceParser, data, [], addEntriesWithNoData); return { isComplete, data }; } export type PColumnKeyList = { /** array of keys */ data: PColumnKey[]; /** length of partition key */ keyLength: number; }; const removeIndexSuffix = (keyStr: string): { baseKey: string; type: "index" | "values" } => { if (keyStr.endsWith(".index")) { return { baseKey: keyStr.substring(0, keyStr.length - 6), type: "index" }; } else if (keyStr.endsWith(".values")) { return { baseKey: keyStr.substring(0, keyStr.length - 7), type: "values" }; } else { throw new Error(`key must ends on .index/.values for binary p-column, got: ${keyStr}`); } }; // @TODO define a class with various resource map operations /** Returns a list of all partition keys appeared in the p-column */ export function getPartitionKeysList( acc: TreeNodeAccessor | undefined, ): PColumnKeyList | undefined { if (!acc) return undefined; const rt = acc.resourceType.name; const meta = acc.getDataAsJson>(); const data: PColumnKey[] = []; let keyLength = 0; // @TODO validate meta shape switch (rt) { case RT_RESOURCE_MAP: keyLength = meta["keyLength"]; break; case RT_RESOURCE_MAP_PARTITIONED: keyLength = meta["partitionKeyLength"] + meta["keyLength"]; break; case RT_JSON_PARTITIONED: case RT_BINARY_PARTITIONED: case RT_PARQUET_PARTITIONED: keyLength = meta["partitionKeyLength"]; break; case RT_BINARY_SUPER_PARTITIONED: case RT_JSON_SUPER_PARTITIONED: case RT_PARQUET_SUPER_PARTITIONED: keyLength = meta["superPartitionKeyLength"] + meta["partitionKeyLength"]; break; } switch (rt) { case RT_RESOURCE_MAP: case RT_JSON_PARTITIONED: case RT_BINARY_PARTITIONED: case RT_PARQUET_PARTITIONED: for (let keyStr of acc.listInputFields()) { if (rt === RT_BINARY_PARTITIONED) { keyStr = removeIndexSuffix(keyStr).baseKey; } const key = [...JSON.parse(keyStr)] as PColumnKey; data.push(key); } break; case RT_RESOURCE_MAP_PARTITIONED: case RT_BINARY_SUPER_PARTITIONED: case RT_JSON_SUPER_PARTITIONED: case RT_PARQUET_SUPER_PARTITIONED: for (const supKeyStr of acc.listInputFields()) { const keyPrefix = [...JSON.parse(supKeyStr)] as PColumnKey; const value = acc.resolve({ field: supKeyStr, assertFieldType: "Input" }); if (value !== undefined) { for (let keyStr of value.listInputFields()) { if (rt === RT_BINARY_SUPER_PARTITIONED) { keyStr = removeIndexSuffix(keyStr).baseKey; } const key = [...keyPrefix, ...JSON.parse(keyStr)] as PColumnKey; data.push(key); } } } break; } return { data, keyLength }; } function getUniquePartitionKeysForDataEntries( list: DataInfoEntries, ): (string | number)[][] { if ( list.type !== "JsonPartitioned" && list.type !== "BinaryPartitioned" && list.type !== "ParquetPartitioned" ) throw new Error(`Splitting requires Partitioned DataInfoEntries, got ${list.type}`); const { parts, partitionKeyLength } = list; const result: Set[] = []; for (let i = 0; i < partitionKeyLength; ++i) { result.push(new Set()); } for (const part of parts) { const key = part.key; if (key.length !== partitionKeyLength) { throw new Error( `Key length (${key.length}) does not match partition length (${partitionKeyLength}) for key: ${JSON.stringify( key, )}`, ); } for (let i = 0; i < partitionKeyLength; ++i) { result[i].add(key[i]); } } return result.map((s) => Array.from(s.values())); } /** Returns an array of unique partition keys for each column: the i-th element in the resulting 2d array contains all unique values of i-th partition axis. */ export function getUniquePartitionKeys(acc: DataInfoEntries): (string | number)[][]; export function getUniquePartitionKeys( acc: DataInfoEntries | TreeNodeAccessor | undefined, ): (string | number)[][] | undefined; export function getUniquePartitionKeys( acc: TreeNodeAccessor | DataInfoEntries | undefined, ): (string | number)[][] | undefined { if (acc === undefined) return undefined; if (isDataInfoEntries(acc)) return getUniquePartitionKeysForDataEntries(acc); const list = getPartitionKeysList(acc); if (!list) return undefined; const { data, keyLength } = list; const result: Set[] = []; for (let i = 0; i < keyLength; ++i) { result.push(new Set()); } for (const l of data) { if (l.length !== keyLength) { throw new Error("key length does not match partition length"); } for (let i = 0; i < keyLength; ++i) { result[i].add(l[i]); } } return result.map((s) => Array.from(s.values())); } /** * Parses the PColumn data from a TreeNodeAccessor into a DataInfoEntries structure. * Returns undefined if any required data is missing. * Throws error on validation failures. * * @param acc - The TreeNodeAccessor containing PColumn data * @param keyPrefix - Optional key prefix for recursive calls * @returns DataInfoEntries representation of the PColumn data, or undefined if incomplete */ export function parsePColumnData( acc: TreeNodeAccessor | undefined, keyPrefix: PColumnKey = [], ): PartitionedDataInfoEntries | undefined { if (acc === undefined) return undefined; if (!acc.getIsReadyOrError()) return undefined; const resourceType = acc.resourceType.name; const meta = acc.getDataAsJson>(); // Prevent recursive super-partitioned resources if ( keyPrefix.length > 0 && (resourceType === RT_JSON_SUPER_PARTITIONED || resourceType === RT_BINARY_SUPER_PARTITIONED || resourceType === RT_PARQUET_SUPER_PARTITIONED) ) { throw new Error(`Unexpected nested super-partitioned resource: ${resourceType}`); } switch (resourceType) { case RT_RESOURCE_MAP: case RT_RESOURCE_MAP_PARTITIONED: throw new Error(`Only data columns are supported, got: ${resourceType}`); case RT_JSON_PARTITIONED: { if (typeof meta?.partitionKeyLength !== "number") { throw new Error(`Missing partitionKeyLength in metadata for ${resourceType}`); } const parts: PColumnDataEntry[] = []; for (const keyStr of acc.listInputFields()) { const value = acc.resolve({ field: keyStr, assertFieldType: "Input" }); if (value === undefined) return undefined; const key = [...keyPrefix, ...JSON.parse(keyStr)]; parts.push({ key, value }); } return { type: "JsonPartitioned", partitionKeyLength: meta.partitionKeyLength, parts, }; } case RT_BINARY_PARTITIONED: { if (typeof meta?.partitionKeyLength !== "number") { throw new Error(`Missing partitionKeyLength in metadata for ${resourceType}`); } const parts: PColumnDataEntry>[] = []; const baseKeys = new Map(); // Group fields by base key (without .index/.values suffix) for (const keyStr of acc.listInputFields()) { const suffix = removeIndexSuffix(keyStr); const value = acc.resolve({ field: keyStr, assertFieldType: "Input" }); if (value === undefined) return undefined; let entry = baseKeys.get(suffix.baseKey); if (!entry) { entry = {}; baseKeys.set(suffix.baseKey, entry); } if (suffix.type === "index") { entry.index = value; } else { entry.values = value; } } // Process complete binary chunks only for (const [baseKeyStr, entry] of baseKeys.entries()) { if (!entry.index || !entry.values) return undefined; const key = [...keyPrefix, ...JSON.parse(baseKeyStr)]; parts.push({ key, value: { index: entry.index, values: entry.values, }, }); } return { type: "BinaryPartitioned", partitionKeyLength: meta.partitionKeyLength, parts, }; } case RT_PARQUET_PARTITIONED: { if (typeof meta?.partitionKeyLength !== "number") { throw new Error(`Missing partitionKeyLength in metadata for ${resourceType}`); } const parts: PColumnDataEntry[] = []; for (const keyStr of acc.listInputFields()) { const value = acc.resolve({ field: keyStr, assertFieldType: "Input" }); if (value === undefined) return undefined; const key = [...keyPrefix, ...JSON.parse(keyStr)]; parts.push({ key, value }); } return { type: "ParquetPartitioned", partitionKeyLength: meta.partitionKeyLength, parts, }; } case RT_JSON_SUPER_PARTITIONED: { if ( typeof meta?.superPartitionKeyLength !== "number" || typeof meta?.partitionKeyLength !== "number" ) { throw new Error( `Missing superPartitionKeyLength or partitionKeyLength in metadata for ${resourceType}`, ); } const totalKeyLength = meta.superPartitionKeyLength + meta.partitionKeyLength; const parts: PColumnDataEntry[] = []; // Process all super partitions for (const supKeyStr of acc.listInputFields()) { const superPartition = acc.resolve({ field: supKeyStr, assertFieldType: "Input" }); if (superPartition === undefined) return undefined; // Validate inner type if (superPartition.resourceType.name !== RT_JSON_PARTITIONED) { throw new Error( `Expected ${RT_JSON_PARTITIONED} inside ${resourceType}, but got ${superPartition.resourceType.name}`, ); } const innerResult = parsePColumnData(superPartition, JSON.parse(supKeyStr) as PColumnKey); if (innerResult === undefined) return undefined; if (innerResult.type !== "JsonPartitioned") throw new Error(`Unexpected inner result type for ${resourceType}: ${innerResult.type}`); parts.push(...innerResult.parts); } return { type: "JsonPartitioned", partitionKeyLength: totalKeyLength, parts, }; } case RT_BINARY_SUPER_PARTITIONED: { if ( typeof meta?.superPartitionKeyLength !== "number" || typeof meta?.partitionKeyLength !== "number" ) { throw new Error( `Missing superPartitionKeyLength or partitionKeyLength in metadata for ${resourceType}`, ); } const totalKeyLength = meta.superPartitionKeyLength + meta.partitionKeyLength; const parts: PColumnDataEntry>[] = []; // Process all super partitions for (const supKeyStr of acc.listInputFields()) { const superPartition = acc.resolve({ field: supKeyStr, assertFieldType: "Input" }); if (superPartition === undefined) return undefined; // Validate inner type if (superPartition.resourceType.name !== RT_BINARY_PARTITIONED) { throw new Error( `Expected ${RT_BINARY_PARTITIONED} inside ${resourceType}, but got ${superPartition.resourceType.name}`, ); } const innerResult = parsePColumnData(superPartition, JSON.parse(supKeyStr) as PColumnKey); if (innerResult === undefined) return undefined; if (innerResult.type !== "BinaryPartitioned") throw new Error(`Unexpected inner result type for ${resourceType}: ${innerResult.type}`); parts.push(...innerResult.parts); } return { type: "BinaryPartitioned", partitionKeyLength: totalKeyLength, parts, }; } case RT_PARQUET_SUPER_PARTITIONED: { if ( typeof meta?.superPartitionKeyLength !== "number" || typeof meta?.partitionKeyLength !== "number" ) { throw new Error( `Missing superPartitionKeyLength or partitionKeyLength in metadata for ${resourceType}`, ); } const totalKeyLength = meta.superPartitionKeyLength + meta.partitionKeyLength; const parts: PColumnDataEntry[] = []; // Process all super partitions for (const supKeyStr of acc.listInputFields()) { const superPartition = acc.resolve({ field: supKeyStr, assertFieldType: "Input" }); if (superPartition === undefined) return undefined; // Validate inner type if (superPartition.resourceType.name !== RT_PARQUET_PARTITIONED) { throw new Error( `Expected ${RT_PARQUET_PARTITIONED} inside ${resourceType}, but got ${superPartition.resourceType.name}`, ); } const innerResult = parsePColumnData(superPartition, JSON.parse(supKeyStr) as PColumnKey); if (innerResult === undefined) return undefined; if (innerResult.type !== "ParquetPartitioned") throw new Error(`Unexpected inner result type for ${resourceType}: ${innerResult.type}`); parts.push(...innerResult.parts); } return { type: "ParquetPartitioned", partitionKeyLength: totalKeyLength, parts, }; } default: throw new Error(`Unknown resource type: ${resourceType}`); } } /** * Converts or parses the input into DataInfoEntries format. * @param acc - The input data, which can be TreeNodeAccessor, DataInfoEntries, DataInfo, or undefined. * @returns The data in DataInfoEntries format, or undefined if the input was undefined or data is not ready. */ export function convertOrParsePColumnData( acc: | TreeNodeAccessor | DataInfoEntries | DataInfo | undefined, ): DataInfoEntries | undefined { if (acc === undefined) return undefined; if (isDataInfoEntries(acc)) return acc; if (isDataInfo(acc)) return dataInfoToEntries(acc); if (acc instanceof TreeNodeAccessor) return parsePColumnData(acc); throw new Error(`Unexpected input type: ${typeof acc}`); } /** * Returns the total number of rows in a PColumn, or `undefined` when the count * cannot be determined without downloading data blobs. * * Derivable cases: * - Inline values ({@link PColumnValues}): one entry per row. * - Inline `Json` {@link DataInfo}: one entry per row. * - Parquet (`ParquetPartitioned` / `ParquetSuperPartitioned`): summed from * each chunk's precomputed {@link ParquetChunkMetadata.stats} `numberOfRows`, * read from the chunk resource metadata without downloading the parquet files. * * Returns `undefined` when: * - the data is `undefined` or still computing / not ready, * - the data is `JsonPartitioned` or `BinaryPartitioned` (per-part row counts * live inside opaque blobs and are not readable here), * - any parquet chunk is missing `stats.numberOfRows`. * * Note: do not confuse row count with `keyLength` / `partitionKeyLength`, which * are axis-tuple lengths, not row counts. */ export function getNumberOfRows(data: PColumnDataUniversal | undefined): number | undefined { if (data === undefined) return undefined; // Inline explicit values: one entry per row. if (Array.isArray(data)) return data.length; const entries = convertOrParsePColumnData(data); if (entries === undefined) return undefined; switch (entries.type) { case "Json": // Inline Json: one entry per row. return entries.data.length; case "JsonPartitioned": case "BinaryPartitioned": // Per-part row counts live inside opaque blobs; not knowable without download. return undefined; case "ParquetPartitioned": { let total = 0; for (const { value } of entries.parts) { // The part accessor points at a ParquetChunk resource whose JSON payload // is ParquetChunkMetadata. Reading it does not download the parquet file // (the file lives in a separate `blob` sub-field of the chunk resource). const rows = value.getDataAsJsonOrUndefined()?.stats?.numberOfRows; // stats / numberOfRows are Partial: if any chunk lacks it, the total is unknowable. if (typeof rows !== "number") return undefined; total += rows; } return total; } default: assertNever(entries); } } export function isPColumnReady( c: PColumn, ): c is PColumn { const isValues = (d: PColumnDataUniversal): d is PColumnValues => Array.isArray(d); const isAccessor = (d: PColumnDataUniversal): d is TreeNodeAccessor => d instanceof TreeNodeAccessor; let ready = true; const data = c.data; if (data == null) { return false; } else if (isAccessor(data)) { ready &&= data.getIsReadyOrError(); } else if (isDataInfo(data)) { visitDataInfo(data, (v) => (ready &&= v.getIsReadyOrError())); } else if (!isValues(data)) { throw Error(`unsupported column data type: ${data satisfies never}`); } return ready; } export function allPColumnsReady( columns: PColumn[], ): columns is PColumn[] { return columns.every(isPColumnReady); }