import type { AnchoredPColumnSelector, AnyFunction, AxisId, DataInfo, ModelServices, Option, PColumn, PColumnSelector, PColumnSpec, PColumnValues, PFrameDef, PFrameHandle, ColumnUniversalId, PObject, PObjectId, PObjectSpec, PTableDef, PTableDefV2, PTableHandle, PTableRecordFilter, PTableSorting, PlRef, ResolveAnchorsOptions, ResultCollection, SUniversalPColumnId, ValueOrError, } from "@milaboratories/pl-model-common"; import { AnchoredIdDeriver, ensurePColumn, parseJson, isDataInfo, isPColumn, isPColumnSpec, isPlRef, mapDataInfo, mapPObjectData, mapPTableDef, mapPTableDefV2, mapValueInVOE, PColumnName, readAnnotation, withEnrichments, legacyColumnSelectorsToPredicate, extractAllColumns, visitDataInfo, } from "@milaboratories/pl-model-common"; import canonicalize from "canonicalize"; import type { Optional } from "utility-types"; import { getCfgRenderCtx } from "../internal"; import { getPluginData } from "../block_storage"; import type { PluginHandle, PluginFactoryLike, InferFactoryData, InferFactoryParams, } from "../plugin_handle"; import { TreeNodeAccessor, ifDef } from "./accessor"; import type { FutureRef } from "./future"; import type { AccessorHandle, GlobalCfgRenderCtx } from "./internal"; import { MainAccessorName, StagingAccessorName } from "./internal"; import { PColumnCollection, type AxisLabelProvider, type ColumnProvider, } from "./util/column_collection"; import type { LabelDerivationOps } from "./util/label"; import { deriveLabels } from "./util/label"; import type { APColumnSelectorWithSplit } from "./util/split_selectors"; import { patchInSetFilters } from "./util/pframe_upgraders"; import type { PColumnDataUniversal } from "./internal"; import { getService } from "../services"; import { allPColumnsReady } from "./util"; import { throwError } from "@milaboratories/helpers"; import { isString } from "es-toolkit"; /** * Helper function to match domain objects * @param query Optional domain to match against * @param target Optional domain to match * @returns true if domains match, false otherwise */ function matchDomain(query?: Record, target?: Record) { if (query === undefined) return target === undefined; if (target === undefined) return true; for (const k in target) { if (query[k] !== target[k]) return false; } return true; } export type UniversalColumnOption = { label: string; value: SUniversalPColumnId }; /** * Transforms PColumn data into the internal representation expected by the platform * @param column Data from a PColumn to transform * @returns Transformed data compatible with platform API */ export function finalizePColumnData( column: PColumn>, ): PColumn> { return mapPObjectData(column, (data) => { if (data instanceof TreeNodeAccessor) { return data.handle; } else if (isDataInfo(data)) { let ready = true; visitDataInfo(data, (accessor) => (ready &&= accessor?.getIsReadyOrError() ?? false)); if (!ready) return []; return mapDataInfo( data, (accessor) => accessor?.handle ?? throwError("Accessor handle is undefined"), ); } else { return data ?? []; } }); } type UniversalPColumnOpts = { labelOps?: LabelDerivationOps; dontWaitAllData?: boolean; exclude?: AnchoredPColumnSelector | AnchoredPColumnSelector[]; } & ResolveAnchorsOptions; type GetOptionsOpts = { /** * If true, references returned by the method will contain `requireEnrichments` flag set to true. * If this reference is added to the block's args, it will communicate to the platform that the block * expects enrichments of the referenced block to be available in the context of the current block. */ refsWithEnrichments?: boolean; /** * Label derivation options. * If provided, it will be used to derive labels for the options. * If not provided, labels will be derived using the default logic. */ label?: ((spec: PObjectSpec, ref: PlRef) => string) | LabelDerivationOps; }; /** * @deprecated use the direct `ctx.*` methods on `RenderCtxBase` instead. * * Migration map: * - `getSpecs` → `ctx.getSpecs` * - `getSpecByRef` / `getPColumnSpecByRef` → `ctx.getSpecByRef` / `ctx.getPColumnSpecByRef` * - `getDataByRef` / `getPColumnByRef` → `ctx.getDataByRef` / `ctx.getPColumnByRef` * - `getOptions` → `ctx.getOptions` * * Eager methods `selectColumns` / `getData` / `getDataWithErrors` should be * replaced by `ctx.getSpecs` + on-demand `ctx.getStatusByRef` / * `ctx.getPColumnByRef` so that the data resource is not subscribed up-front. */ export class ResultPool implements ColumnProvider, AxisLabelProvider { private readonly ctx: GlobalCfgRenderCtx = getCfgRenderCtx(); public getOptions( predicateOrSelector: ((spec: PObjectSpec) => boolean) | PColumnSelector | PColumnSelector[], opts?: GetOptionsOpts, ): Option[]; /** @deprecated wrap label ops with { label: <...> } */ public getOptions( predicateOrSelector: ((spec: PObjectSpec) => boolean) | PColumnSelector | PColumnSelector[], label?: ((spec: PObjectSpec, ref: PlRef) => string) | LabelDerivationOps, ): Option[]; public getOptions( predicateOrSelector: ((spec: PObjectSpec) => boolean) | PColumnSelector | PColumnSelector[], opts?: GetOptionsOpts | ((spec: PObjectSpec, ref: PlRef) => string) | LabelDerivationOps, ): Option[] { const predicate = typeof predicateOrSelector === "function" ? predicateOrSelector : legacyColumnSelectorsToPredicate(predicateOrSelector); const filtered = this.getSpecs().entries.filter((s) => predicate(s.obj)); let labelOps: LabelDerivationOps | ((spec: PObjectSpec, ref: PlRef) => string) = {}; let refsWithEnrichments: boolean = false; if (typeof opts !== "undefined") { if (typeof opts === "function") { labelOps = opts; } else if (typeof opts === "object") { if ("includeNativeLabel" in opts || "separator" in opts || "addLabelAsSuffix" in opts) { labelOps = opts; } else { opts = opts as GetOptionsOpts; labelOps = opts.label ?? {}; refsWithEnrichments = opts.refsWithEnrichments ?? false; } } } if (typeof labelOps === "object") return deriveLabels(filtered, (o) => o.obj, labelOps ?? {}).map( ({ value: { ref }, label }) => ({ ref: withEnrichments(ref, refsWithEnrichments), label, }), ); else return filtered.map(({ ref, obj }) => ({ ref: withEnrichments(ref, refsWithEnrichments), label: labelOps(obj, ref), })); } public resolveAnchorCtx( anchorsOrCtx: AnchoredIdDeriver | Record, ): AnchoredIdDeriver | undefined { if (anchorsOrCtx instanceof AnchoredIdDeriver) return anchorsOrCtx; const resolvedAnchors: Record = {}; for (const [key, value] of Object.entries(anchorsOrCtx)) { if (isPlRef(value)) { const resolvedSpec = this.getPColumnSpecByRef(value); if (!resolvedSpec) return undefined; resolvedAnchors[key] = resolvedSpec; } else { resolvedAnchors[key] = value; } } return new AnchoredIdDeriver(resolvedAnchors); } /** * Returns columns that match the provided anchors and selectors. It applies axis filters and label derivation. * * @param anchorsOrCtx - Anchor context for column selection (same as in getCanonicalOptions) * @param predicateOrSelectors - Predicate or selectors for filtering columns (same as in getCanonicalOptions) * @param opts - Optional configuration for label generation and data waiting * @returns A PFrameHandle for the created PFrame, or undefined if any required data is missing */ public getAnchoredPColumns( anchorsOrCtx: AnchoredIdDeriver | Record, predicateOrSelectors: | ((spec: PColumnSpec) => boolean) | APColumnSelectorWithSplit | APColumnSelectorWithSplit[], opts?: UniversalPColumnOpts, ): undefined | PColumn[] { const anchorCtx = this.resolveAnchorCtx(anchorsOrCtx); if (!anchorCtx) return undefined; return new PColumnCollection() .addColumnProvider(this) .addAxisLabelProvider(this) .getColumns(predicateOrSelectors, { ...opts, anchorCtx, }); } /** * Calculates anchored identifier options for columns matching a given predicate and returns their * canonicalized representations. * * This function filters column specifications from the result pool that match the provided predicate, * creates a standardized AnchorCtx from the provided anchors, and generates a list of label-value * pairs for UI components (like dropdowns). * * @param anchorsOrCtx - Either: * - An existing AnchorCtx instance * - A record mapping anchor IDs to PColumnSpec objects * - A record mapping anchor IDs to PlRef objects (which will be resolved to PColumnSpec) * @param predicateOrSelectors - Either: * - A predicate function that takes a PColumnSpec and returns a boolean. * Only specs that return true will be included. * - An APColumnSelector object for declarative filtering, which will be * resolved against the provided anchors and matched using matchPColumn. * - An array of APColumnSelector objects - columns matching ANY selector * in the array will be included (OR operation). * @param opts - Optional configuration for label generation: * - labelOps: Optional configuration for label generation: * - includeNativeLabel: Whether to include native column labels * - separator: String to use between label parts (defaults to " / ") * - addLabelAsSuffix: Whether to add labels as suffix instead of prefix * - dontWaitAllData: Whether to skip columns that don't have all data (if not set, will return undefined, * if at least one column that requires splitting is missing data) * @returns An array of objects with `label` (display text) and `value` (anchored ID string) properties, * or undefined if any PlRef resolution fails. */ getCanonicalOptions( anchorsOrCtx: AnchoredIdDeriver | Record, predicateOrSelectors: | ((spec: PColumnSpec) => boolean) | APColumnSelectorWithSplit | APColumnSelectorWithSplit[], opts?: UniversalPColumnOpts, ): { label: string; value: SUniversalPColumnId }[] | undefined { const anchorCtx = this.resolveAnchorCtx(anchorsOrCtx); if (!anchorCtx) return undefined; const entries = new PColumnCollection() .addColumnProvider(this) .addAxisLabelProvider(this) .getUniversalEntries(predicateOrSelectors, { ...opts, anchorCtx, }); if (!entries) return undefined; return entries.map((item) => ({ value: item.id, label: item.label, })); } /** * @deprecated use getData() */ public getDataFromResultPool(): ResultCollection> { return this.getData(); } public getData(): ResultCollection> { const result = this.ctx.getDataFromResultPool(); return { isComplete: result.isComplete, entries: result.entries.map((e) => ({ ref: e.ref, obj: { ...e.obj, data: new TreeNodeAccessor(e.obj.data, [e.ref.blockId, e.ref.name]), }, })), }; } /** * @deprecated use getDataWithErrors() */ public getDataWithErrorsFromResultPool(): ResultCollection< Optional>, "id"> > { return this.getDataWithErrors(); } public getDataWithErrors(): ResultCollection< Optional>, "id"> > { const result = this.ctx.getDataWithErrorsFromResultPool(); return { isComplete: result.isComplete, entries: result.entries.map((e) => ({ ref: e.ref, obj: { ...e.obj, data: mapValueInVOE( e.obj.data, (handle) => new TreeNodeAccessor(handle, [e.ref.blockId, e.ref.name]), ), }, })), }; } /** * @deprecated use getSpecs() */ public getSpecsFromResultPool(): ResultCollection { return this.getSpecs(); } public getSpecs(): ResultCollection { return this.ctx.getSpecsFromResultPool(); } /** * @param ref a Ref * @returns data associated with the ref */ public getDataByRef(ref: PlRef): PObject | undefined { const data = this.ctx.getDataFromResultPoolByRef(ref.blockId, ref.name); // Keep original call // Need to handle undefined case before mapping if (!data) return undefined; return mapPObjectData(data, (handle) => new TreeNodeAccessor(handle, [ref.blockId, ref.name])); } /** * Returns data associated with the ref ensuring that it is a p-column. * @param ref a Ref * @returns p-column associated with the ref */ public getPColumnByRef(ref: PlRef): PColumn | undefined { const data = this.getDataByRef(ref); if (!data) return undefined; return ensurePColumn(data); } /** * Returns spec associated with the ref ensuring that it is a p-column spec. * @param ref a Ref * @returns p-column spec associated with the ref */ public getPColumnSpecByRef(ref: PlRef): PColumnSpec | undefined { const spec = this.getSpecByRef(ref); if (!spec) return undefined; if (!isPColumnSpec(spec)) throw new Error(`not a PColumn spec (kind = ${spec.kind})`); return spec; } /** * @param ref a Ref * @returns object spec associated with the ref */ public getSpecByRef(ref: PlRef): PObjectSpec | undefined { return this.ctx.getSpecFromResultPoolByRef(ref.blockId, ref.name); } /** * @param spec object specification * @returns array of data objects with compatible specs * @deprecated delete this method after Jan 1, 2025 */ public findDataWithCompatibleSpec(spec: PColumnSpec): PObject[] { const result: PObject[] = []; out: for (const data of this.getData().entries) { if (!isPColumnSpec(data.obj.spec)) { continue; } const oth = data.obj.spec; if (spec.name !== oth.name) { continue; } if (spec.valueType !== oth.valueType) { continue; } if (spec.axesSpec.length !== oth.axesSpec.length) { continue; } if (!matchDomain(spec.domain, oth.domain)) { continue; } if (!matchDomain(spec.contextDomain, oth.contextDomain)) { continue; } for (let i = 0; i < spec.axesSpec.length; ++i) { const qAx = spec.axesSpec[i]; const tAx = oth.axesSpec[i]; if (qAx.name !== tAx.name) { continue out; } if (qAx.type !== tAx.type) { continue out; } if (!matchDomain(qAx.domain, tAx.domain)) { continue out; } if (!matchDomain(qAx.contextDomain, tAx.contextDomain)) { continue out; } } result.push(data.obj); } return result; } /** * Find labels data for a given axis id. It will search for a label column and return its data as a map. * @returns a map of axis value => label */ public findLabels(axis: AxisId): Record | undefined { const dataPool = this.getData(); for (const column of dataPool.entries) { if (!isPColumn(column.obj)) continue; const spec = column.obj.spec; if ( spec.name === PColumnName.Label && spec.axesSpec.length === 1 && spec.axesSpec[0].name === axis.name && spec.axesSpec[0].type === axis.type && matchDomain(axis.domain, spec.axesSpec[0].domain) && matchDomain(axis.contextDomain, spec.axesSpec[0].contextDomain) ) { if (column.obj.data.resourceType.name !== "PColumnData/Json") { throw Error(`Expected JSON column for labels, got: ${column.obj.data.resourceType.name}`); } const labels: Record = Object.fromEntries( Object.entries( column.obj.data.getDataAsJson<{ data: Record; }>().data, ).map((e) => [JSON.parse(e[0])[0], e[1]]), ); return labels; } } return undefined; } /** * Selects columns based on the provided selectors, returning PColumn objects * with lazily loaded data. * * @param selectors - A predicate function, a single selector, or an array of selectors. * @returns An array of PColumn objects matching the selectors. Data is loaded on first access. * @deprecated use `RenderCtxBase.rawResultPool` + `ResultPoolColumnsProvider` + * `discoverColumns` instead. This eager path materializes column data * unnecessarily for callers that only filter by spec. */ public selectColumns( selectors: ((spec: PColumnSpec) => boolean) | PColumnSelector | PColumnSelector[], ): PColumn[] { const predicate = typeof selectors === "function" ? selectors : legacyColumnSelectorsToPredicate(selectors); const matchedSpecs = this.getSpecs().entries.filter(({ obj: spec }) => { if (!isPColumnSpec(spec)) return false; return predicate(spec); }); // Map specs to PColumn objects with lazy data loading return matchedSpecs.map(({ ref, obj: spec }) => { // Type assertion needed because filter ensures it's PColumnSpec const pcolumnSpec = spec as PColumnSpec; let _cachedData: TreeNodeAccessor | undefined | null = null; // Use null to distinguish initial state from undefined result // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; // Capture 'this' for use inside the getter return { id: canonicalize(ref) as PObjectId, spec: pcolumnSpec, get data(): TreeNodeAccessor | undefined { if (_cachedData !== null) { return _cachedData; // Return cached data (could be undefined if fetch failed) } _cachedData = self.getPColumnByRef(ref)?.data; return _cachedData; }, } satisfies PColumn; // Cast needed because 'data' is a getter }); } /** * Find labels data for a given axis id of a p-column. * @returns a map of axis value => label */ public findLabelsForColumnAxis( column: PColumnSpec, axisIdx: number, ): Record | undefined { const labels = this.findLabels(column.axesSpec[axisIdx]); if (!labels) return undefined; const axisKeys = readAnnotation(column, `pl7.app/axisKeys/${axisIdx}`); if (axisKeys !== undefined) { const keys = JSON.parse(axisKeys) as string[]; return Object.fromEntries( keys.map((key) => { return [key, labels[key] ?? "Unlabelled"]; }), ); } else { return labels; } } } /** Main entry point to the API available within model lambdas (like outputs, sections, etc..) */ export abstract class RenderCtxBase { public readonly ctx: GlobalCfgRenderCtx; constructor() { this.ctx = getCfgRenderCtx(); } private dataCache?: { v: Data }; public get data(): Data { if (this.dataCache === undefined) { const raw = this.ctx.data; const value = typeof raw === "function" ? raw() : raw; this.dataCache = { v: value ? JSON.parse(value) : ({} as Data) }; } return this.dataCache.v; } // lazy rendering because this feature is rarely used private activeArgsCache?: { v?: Args }; /** * Returns args snapshot the block was executed for (i.e. when "Run" button was pressed). * Returns undefined, if block was never executed or stopped mid-way execution, so that the result was cleared. * */ public get activeArgs(): Args | undefined { if (this.activeArgsCache === undefined) { const raw = this.ctx.activeArgs; const value = typeof raw === "function" ? raw() : raw; this.activeArgsCache = { v: value ? JSON.parse(value) : undefined, }; } return this.activeArgsCache.v; } public getService(name: T): ModelServices[T] { return getService(name); } // /** Can be used to determine features provided by the desktop instance. */ // public get featureFlags() { // return this.ctx.featureFlags; // } private getNamedAccessor(name: string): TreeNodeAccessor | undefined { return ifDef( this.ctx.getAccessorHandleByName(name), (accessor) => new TreeNodeAccessor(accessor, [name]), ); } public get prerun(): TreeNodeAccessor | undefined { return this.getNamedAccessor(StagingAccessorName); } public get outputs(): TreeNodeAccessor | undefined { return this.getNamedAccessor(MainAccessorName); } /** @deprecated use `rawResultPool` + `ResultPoolColumnsProvider` instead. */ public readonly resultPool = new ResultPool(); private patchPTableDef( def: PTableDef>, ): PTableDef> { if (!this.ctx.featureFlags?.pTablePartitionFiltersSupport) { // For old desktop move all partition filters to filters field as it doesn't read partitionFilters field def = { ...def, partitionFilters: [], filters: [...def.partitionFilters, ...def.filters], }; } if (!this.ctx.featureFlags?.pFrameInSetFilterSupport) { def = { ...def, partitionFilters: patchInSetFilters(def.partitionFilters), filters: patchInSetFilters(def.filters), }; } return def; } public createPFrame( def: PFrameDef>, ): PFrameHandle | undefined { return this.ctx.createPFrame(def.map((v) => (isString(v) ? v : finalizePColumnData(v)))); } // TODO remove all non-PColumn fields public createPTable(def: PTableDef>): PTableHandle | undefined; public createPTable(def: { columns: PColumn[]; filters?: PTableRecordFilter[]; /** Table sorting */ sorting?: PTableSorting[]; }): PTableHandle | undefined; public createPTable( def: | PTableDef> | { columns: PColumn[]; filters?: PTableRecordFilter[]; /** Table sorting */ sorting?: PTableSorting[]; }, ): PTableHandle | undefined { let rawDef: PTableDef>; if ("columns" in def) { rawDef = this.patchPTableDef({ src: { type: "full", entries: def.columns.map((c) => ({ type: "column", column: c })), }, partitionFilters: def.filters ?? [], filters: [], sorting: def.sorting ?? [], }); } else { rawDef = this.patchPTableDef(def); } const columns = extractAllColumns(rawDef.src); if (!allPColumnsReady(columns)) return undefined; return this.ctx.createPTable(mapPTableDef(rawDef, finalizePColumnData)); } public createPTableV2( def: PTableDefV2>, ): PTableHandle | undefined { return this.ctx.createPTableV2( mapPTableDefV2(def, { column: (v) => (isString(v) ? v : finalizePColumnData(v)) }), ); } public getCurrentUnstableMarker(): string | undefined { return this.ctx.getCurrentUnstableMarker(); } public logInfo(msg: string): void { this.ctx.logInfo(msg); } public logWarn(msg: string): void { this.ctx.logWarn(msg); } public logError(msg: string): void { this.ctx.logError(msg); } } /** Main entry point to the API available within model lambdas (like outputs, sections, etc..) for v3+ blocks */ export class BlockRenderCtx extends RenderCtxBase { private argsCache?: { v: Args | undefined }; public get args(): Args | undefined { if (this.argsCache === undefined) { const raw = this.ctx.args; const value = typeof raw === "function" ? raw() : raw; this.argsCache = { v: value === undefined ? undefined : JSON.parse(value) }; } return this.argsCache.v; } } /** * Render context for plugin output functions. * Reads plugin data from blockStorage and derives params from pre-wrapped input callbacks. * * Parameterized on the factory-like phantom F so that getPluginData returns * InferFactoryData directly — no casts needed for the data getter. * * @typeParam F - PluginFactoryLike phantom carrying data/params/outputs types */ export class PluginRenderCtx extends RenderCtxBase< unknown, InferFactoryData > { private readonly handle: PluginHandle; private readonly wrappedInputs: Record unknown>; constructor(handle: PluginHandle, wrappedInputs: Record unknown>) { super(); this.handle = handle; this.wrappedInputs = wrappedInputs; } private pluginDataCache?: { v: InferFactoryData }; /** Plugin's persistent data from blockStorage.__plugins.{pluginId}.__data */ public override get data(): InferFactoryData { if (this.pluginDataCache === undefined) { const raw = this.ctx.blockStorage(); this.pluginDataCache = { v: getPluginData(parseJson(raw), this.handle) }; } return this.pluginDataCache.v; } private paramsCache?: { v: InferFactoryParams }; /** Params derived from block context via pre-wrapped input callbacks */ public get params(): InferFactoryParams { if (this.paramsCache === undefined) { const result: Record = {}; for (const [key, fn] of Object.entries(this.wrappedInputs)) { result[key] = fn(); } this.paramsCache = { v: result as InferFactoryParams }; } return this.paramsCache.v; } } /** @deprecated Use BlockRenderCtx instead */ export type RenderCtx = BlockRenderCtx; export type RenderFunction = ( rCtx: BlockRenderCtx, ) => Ret; export type UnwrapFutureRef = K extends FutureRef ? T : K extends bigint | boolean | null | number | string | symbol | undefined ? K : { [key in keyof K]: UnwrapFutureRef }; export type InferRenderFunctionReturn = RF extends (...args: any) => infer R ? UnwrapFutureRef : never;