/* eslint-disable max-len */ import * as grok from 'datagrok-api/grok'; import * as ui from 'datagrok-api/ui'; import * as DG from 'datagrok-api/dg'; import {injectTreeForGridUI2} from '../viewers/inject-tree-for-grid2'; import {DistanceMetric, isLeaf, LinkageMethod, NodeType} from '@datagrok-libraries/bio/src/trees'; import {TreeHelper} from './tree-helper'; import {getClusterMatrixWorker} from '@datagrok-libraries/math'; import {ITreeHelper} from '@datagrok-libraries/bio/src/trees/tree-helper'; import {attachLoaderDivToGrid} from '.'; import {GridNeighbor} from '@datagrok-libraries/gridext/src/ui/GridNeighbor'; export const DENDROGRAM_NEIGHBOR_TEMP_NAME = '__dendrogram_neighbor_temp__'; // Custom UI Dialog for Hierarchical Clustering export async function hierarchicalClusteringDialog(getDefaultCoulumn: (t: DG.DataFrame) => DG.Column | null = (_t) => null): Promise { if (!grok.shell.tv?.table) { grok.shell.warning('Please open a table for hierarchical clustering.'); return; } let currentTableView = grok.shell.tv.table; let currentSelectedColNames: string[] = []; const availableColNames = (table: DG.DataFrame): string[] => { return table.columns.toList() .filter((col) => col.type === DG.TYPE.FLOAT || col.type === DG.TYPE.INT || col.semType === DG.SEMTYPE.MACROMOLECULE || col.semType === DG.SEMTYPE.MOLECULE ).map((col) => col.name); }; const onColNamesChange = (columns: DG.Column[]) => { currentSelectedColNames = columns.map((c) => c.name); }; const onTableInputChanged = (table: DG.DataFrame) => { const defaultCol = getDefaultCoulumn(table); const defaultVal = defaultCol ? [defaultCol] : []; const newColInput = ui.input.columns('Features', {table: table, onValueChanged: (value) => onColNamesChange(value), available: availableColNames(table), value: defaultVal}); ui.empty(columnsInputDiv); columnsInputDiv.appendChild(newColInput.root); currentTableView = table; currentSelectedColNames = newColInput.value.map((c) => c.name); }; const tableInput = ui.input.table('Table', {value: currentTableView!, items: grok.shell.tables, nullable: false, onValueChanged: (value) => onTableInputChanged(value)}); const columnsInput = ui.input.columns('Features', {table: currentTableView!, onValueChanged: (value) => onColNamesChange(value), available: availableColNames(currentTableView!)}); const columnsInputDiv = ui.div([columnsInput]); const distanceInput = ui.input.choice('Distance', {value: DistanceMetric.Euclidean, items: Object.values(DistanceMetric)}); const linkageInput = ui.input.choice('Linkage', {value: LinkageMethod.Ward, items: Object.values(LinkageMethod)}); const verticalDiv = ui.divV([ tableInput.root, columnsInputDiv, distanceInput.root, linkageInput.root, ]); onTableInputChanged(currentTableView!); ui.dialog('Hierarchical Clustering') .add(verticalDiv) .show() .onOK(async () => { const pi = DG.TaskBarProgressIndicator.create('Creating dendrogram ...'); try { await hierarchicalClusteringUI(currentTableView!, currentSelectedColNames, distanceInput.value!, linkageInput.value!); } finally { pi.close(); } }); } // Cretes and injects dendrogram to the grid export async function hierarchicalClusteringUI( df: DG.DataFrame, colNameList: string[], distance: DistanceMetric = DistanceMetric.Euclidean, linkage: string, neighborWidth: number = 300, options?: {tableView?: DG.TableView} ): Promise { const linkageCode = Object.values(LinkageMethod).findIndex((method) => method === linkage); const colNameSet: Set = new Set(colNameList); const [filteredDf, filteredIndexList]: [DG.DataFrame, Int32Array] = hierarchicalClusteringFilterDfForNulls(df, colNameSet); const th: ITreeHelper = new TreeHelper(); let tv: DG.TableView = options ? options.tableView ?? grok.shell.getTableView(df.name) : grok.shell.getTableView(df.name); if (filteredDf.rowCount != df.rowCount) { grok.shell.warning('Hierarchical clustering analysis on data filtered out for nulls.'); tv = grok.shell.addTableView(filteredDf); } if (!tv.grid) throw new Error('TableView has no grid to attach dendrogram to.'); if (tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME]) { grok.shell.warning('Dendrogram is already attached to the table grid. Closing existing dendrogram.'); const existingNB: GridNeighbor = tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME]; if (existingNB && typeof existingNB.close === 'function') { try { existingNB.close(); } catch (_) { } } tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME] = null; } const loaderNB = attachLoaderDivToGrid(tv.grid, neighborWidth); let loaderClosed = false; const closeLoader = () => { if (loaderClosed) return; loaderClosed = true; try { loaderNB.close(); } catch {} }; // TODO: Filter rows with nulls in selected columns const preparedDf = DG.DataFrame.fromColumns( filteredDf.columns.toList() .filter((col) => colNameSet.has(col.name)) .map((col) => { let res: DG.Column; switch (col.type) { case DG.COLUMN_TYPE.DATE_TIME: // column of type 'datetime' getRawData() returns Float64Array const colData: Float64Array = col.getRawData() as Float64Array; res = DG.Column.float(col.name, col.length).init((rowI) => { return !col.isNone(rowI) ? colData[rowI] : null; }); break; default: res = col; } return res; })); try { const distanceMatrix = await th.calcDistanceMatrix(preparedDf, preparedDf.columns.toList().map((col) => col.name), distance); const clusterMatrixWorker = getClusterMatrixWorker( distanceMatrix!.data, preparedDf.rowCount, linkageCode ); const clusterMatrix = await clusterMatrixWorker; // const hcPromise = hierarchicalClusteringByDistanceExec(distanceMatrix!, linkage); // Replace rows indexes with filtered // newickStr returned with row indexes after filtering, so we need reversed dict { [fltIdx: number]: number} const fltRowIndexes: { [fltIdx: number]: number } = {}; const fltRowCount: number = filteredDf.rowCount; for (let fltRowIdx: number = 0; fltRowIdx < fltRowCount; fltRowIdx++) fltRowIndexes[fltRowIdx] = filteredIndexList[fltRowIdx]; const newickRoot: NodeType = th.parseClusterMatrix(clusterMatrix); // Fix branch_length for root node as required for hierarchical clustering result newickRoot.branch_length = 0; (function replaceNodeName(node: NodeType, fltRowIndexes: { [fltIdx: number]: number }) { if (!isLeaf(node)) { for (const childNode of node.children!) replaceNodeName(childNode, fltRowIndexes); } })(newickRoot, fltRowIndexes); // empty clusterDf to stub injectTreeForGridUI2 // const clusterDf = DG.DataFrame.fromColumns([ // DG.Column.fromList(DG.COLUMN_TYPE.STRING, 'cluster', [])]); closeLoader(); // bail out if the table view was disposed while we were awaiting compute if (!tv?.grid || !tv?.grid?.dataFrame) return; tv.grid.props.onInitializedScript = ` setTimeout(async () => { const t = grok.shell.table('${tv.dataFrame.name}'); if (!t) return; await t.meta.detectSemanticTypes(); const func = DG.Func.find({name: 'hierarchicalClustering'})[0]; if (!func) return; const cols = ${JSON.stringify(colNameList)}; func.apply({ df: t, colNameList: cols, distance: '${distance}', linkage: '${linkage}' }); }, 1000) `; const nb = injectTreeForGridUI2(tv.grid, newickRoot, undefined, neighborWidth, undefined, {colNames: colNameList, distance}); const s = nb.onClosed.subscribe(() => { tv.grid.props.onInitializedScript = ''; if (tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME] === nb) tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME] = null; s.unsubscribe(); }); const viewRemoveSub = grok.events.onViewRemoved.subscribe((view) => { if (view === tv) { try { viewRemoveSub.unsubscribe(); closeLoader(); tv.grid.props.onInitializedScript = ''; } catch {} }; }); tv.grid.invalidate(); tv.grid.temp[DENDROGRAM_NEIGHBOR_TEMP_NAME] = nb; } catch (err) { grok.shell.error('Error during hierarchical clustering. See console for details.'); console.error(err); try { tv?.grid?.invalidate(); } catch {} closeLoader(); } } export function hierarchicalClusteringFilterDfForNulls( df: DG.DataFrame, colNameSet: Set ): [DG.DataFrame, Int32Array] { // filteredNullsDf to open new table view const colList: DG.Column[] = df.columns.toList().filter((col) => colNameSet.has(col.name)); const filter: DG.BitSet = DG.BitSet.create(df.rowCount, (rowI: number) => { // TODO: Check nulls in columns of colNameList return colList.every((col) => !col.isNone(rowI)); }); const filteredDf: DG.DataFrame = df.clone(filter); const filteredIndexList: Int32Array = filter.getSelectedIndexes(); return [filteredDf, filteredIndexList]; }