// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ // ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ // ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ // ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ // ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ // ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ // ┃ Copyright (c) 2017, the Perspective Authors. ┃ // ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ // ┃ This file is part of the Perspective library, distributed under the terms ┃ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ /** * An implementation of a Perspective Virtual Server for DuckDB. * * This import is optional, and so must be imported manually from either * `@perspective-dev/client/dist/esm/virtual_servers/duckdb.js` or * `@perspective-dev/client/src/ts/virtual_servers/duckdb.ts`, it is not * exported from the package root `@perspective-dev/client` * * @module */ import type * as perspective from "@perspective-dev/client"; import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; import type * as clickhouse from "@clickhouse/client-web"; const NUMBER_AGGS = [ "sum", "count", "any_value", "arbitrary", "array_agg", "avg", "bit_and", "bit_or", "bit_xor", "bitstring_agg", "bool_and", "bool_or", "countif", "favg", "fsum", "geomean", "kahan_sum", "last", "max", "min", "product", "string_agg", "sumkahan", ]; const STRING_AGGS = [ "count", "any_value", "arbitrary", "first", "countif", "last", "string_agg", ]; const FILTER_OPS = [ "==", "!=", "LIKE", "IS DISTINCT FROM", "IS NOT DISTINCT FROM", ">=", "<=", ">", "<", ]; function duckdbTypeToPsp(name: string): ColumnType { if (name.startsWith("Nullable")) { name = name.match(/Nullable\((.+?)\)/)![1]; } if (name.startsWith("Array")) { return "string"; } if (name === "Int64" || name === "UInt64" || name === "Float64") { return "float"; } if (name === "String") { return "string"; } if (name === "DateTime") { return "datetime"; } if (name === "Date") { return "date"; } throw new Error(`Unknown type '${name}'`); } function convertDecimalToNumber(value: any, dtypeString: string) { if (!(value instanceof Uint32Array || value instanceof Int32Array)) { return value; } let bigIntValue = BigInt(0); for (let i = 0; i < value.length; i++) { bigIntValue |= BigInt(value[i]) << BigInt(i * 32); } const scaleMatch = dtypeString.match(/Decimal\[\d+e(\d+)\]/); if (scaleMatch) { const scale = parseInt(scaleMatch[1]); return Number(bigIntValue) / Math.pow(10, scale); } else { return Number(bigIntValue); } } class Lock { lockPromise: Promise; constructor() { this.lockPromise = Promise.resolve(); } acquire() { let releaseLock: (value: void) => void; const newLockPromise: Promise = new Promise((resolve) => { releaseLock = resolve; }); const acquirePromise = this.lockPromise.then(() => releaseLock); this.lockPromise = newLockPromise; return acquirePromise; } } const LOCK = new Lock(); async function runQuery( db: clickhouse.ClickHouseClient, query: string, options: { columns?: true; execute?: boolean }, ): Promise<{ rows: any[]; columns: string[]; dtypes: string[]; }>; async function runQuery( db: clickhouse.ClickHouseClient, query: string, options?: { columns?: false; execute?: boolean }, ): Promise; async function runQuery( db: clickhouse.ClickHouseClient, query: string, options: { columns?: boolean; execute?: boolean } = {}, ) { query = query.replace(/\s+/g, " ").trim(); const release = await LOCK.acquire(); try { const result = await db.query({ query }); if (!options.execute) { const { data, meta } = (await result.json()) as clickhouse.ResponseJSON; if (options.columns) { return { rows: data, columns: meta!.map((f) => f.name), dtypes: meta!.map((f) => f.type), }; } return data; } } catch (error) { console.error("Query error:", error); console.error("Query:", query); throw error; } finally { release(); } } /** * An implementation of Perspective's Virtual Server for `@duckdb/duckdb-wasm`. */ export class ClickhouseHandler implements perspective.VirtualServerHandler { private db: clickhouse.ClickHouseClient; private sqlBuilder: perspective.GenericSQLVirtualServerModel; constructor(db: clickhouse.ClickHouseClient, mod?: typeof perspective) { if (!mod) { if (customElements) { const viewer_class: any = customElements.get("perspective-viewer"); if (viewer_class) { mod = viewer_class.__wasm_module__; } else { throw new Error("Missing perspective-client.wasm"); } } else { } } this.db = db; this.sqlBuilder = new mod!.GenericSQLVirtualServerModel({ create_entity: "VIEW", grouping_fn: "GROUPING", }); } getFeatures() { return { group_by: true, split_by: false, sort: true, expressions: true, group_rollup_mode: ["rollup", "flat", "total"], filter_ops: { integer: FILTER_OPS, float: FILTER_OPS, string: FILTER_OPS, boolean: FILTER_OPS, date: FILTER_OPS, datetime: FILTER_OPS, }, aggregates: { integer: NUMBER_AGGS, float: NUMBER_AGGS, string: STRING_AGGS, boolean: STRING_AGGS, date: STRING_AGGS, datetime: STRING_AGGS, }, }; } async getHostedTables() { const query = "SHOW TABLES"; const results = await runQuery(this.db, query); return results.map((row) => { return `${row.name}`; }); } async tableSchema(tableId: string, config?: ViewConfig) { const query = this.sqlBuilder.tableSchema(tableId); const results = await runQuery(this.db, query); const schema = {} as Record; for (const result of results) { const colName = result.name; if (!colName.startsWith("__")) { schema[colName] = duckdbTypeToPsp(result.type) as ColumnType; } } return schema; } async viewColumnSize(viewId: string, config: ViewConfig) { const query = `SELECT COUNT() FROM system.columns WHERE table = '${viewId}'`; const results = await runQuery(this.db, query); const gs = config.group_by?.length || 0; const count = Number(results[0]["COUNT()"]); console.log(count); return ( count - (gs === 0 ? 0 : gs + (config.split_by?.length === 0 ? 1 : 0)) ); } async tableSize(tableId: string) { const query = this.sqlBuilder.tableSize(tableId); const results = await runQuery(this.db, query); return Number(results[0]["COUNT()"]); } async tableMakeView(tableId: string, viewId: string, config: ViewConfig) { const query = this.sqlBuilder.tableMakeView(tableId, viewId, config); await runQuery(this.db, query, { execute: true }); } async tableValidateExpression(tableId: string, expression: string) { const query = this.sqlBuilder.tableValidateExpression( tableId, expression, ); const results = await runQuery(this.db, query); return duckdbTypeToPsp(results[0]["type"]) as ColumnType; } async viewDelete(viewId: string) { const query = this.sqlBuilder.viewDelete(viewId); await runQuery(this.db, query, { execute: true }); } async viewGetData( viewId: string, config: ViewConfig, schema: Record, viewport: ViewWindow, dataSlice: perspective.VirtualDataSlice, ) { const query = this.sqlBuilder.viewGetData( viewId, config, viewport, schema, ); const { rows, columns, dtypes } = await runQuery(this.db, query, { columns: true, }); for (let cidx = 0; cidx < columns.length; cidx++) { const col = columns[cidx]; const dtype = duckdbTypeToPsp(dtypes[cidx]) as ColumnType; const isDecimal = dtypes[cidx].startsWith("Decimal"); for (let ridx = 0; ridx < rows.length; ridx++) { const row = rows[ridx]; const grouping_id = row["__GROUPING_ID__"]; let value = row[col]; if (isDecimal) { value = convertDecimalToNumber(value, dtypes[cidx]); } if (typeof value === "bigint") { value = Number(value); } if (dtype === "datetime" && typeof value === "string") { value = +new Date(value); } if (dtype === "string" && typeof value !== "string") { value = `${value}`; } dataSlice.setCol(dtype, col, ridx, value, grouping_id); } } } }