/* eslint-disable @typescript-eslint/no-explicit-any */ import { ClientType } from './types.js'; import { File, FileType, EdgeFile, NodeFile } from './File.js'; import { Mode, ModeAction, Privacy } from './Privacy.js'; /** * # Dataset examples * * Datasets are how to combine files into a single visualizable graph. * Powerfully, you can also specify visualization settings and * data-driven visual encodings as part of your dataset's bindings. * * For the many options, see the [JSON documentation](https://hub.graphistry.com/docs/api/). * *
* * --- * *
* * @example **Create a dataset from edges and upload using async/await** * ```javascript * import { Dataset } from '@graphistry/node-api'; * const dataset = new Dataset( * { * node_encodings: { bindings: { } }, * edge_encodings: { bindings: { source: 's', destination: 'd' } }, * metadata: {}, * name: 'testdata', * }, * edgesFile * ); * await dataset.upload(client); * console.log(`Dataset ${dataset.datasetID} uploaded to ${dataset.datasetURL}`); * ``` * *
* * @example **Create a dataset from nodes + edges and upload using promises** * ```javascript * import { Dataset } from '@graphistry/node-api'; * const dataset = new Dataset( * { * node_encodings: { bindings: { 'node': 'n' } }, * edge_encodings: { bindings: { 'source': 's', 'destination': 'd' } }, * metadata: {}, * name: 'testdata', * }, * edgesFile, * nodesFile * ); * dataset.upload(client).then( * () => console.log(`Dataset ${dataset.datasetID} uploaded to ${dataset.datasetURL}`) * ).catch(err => console.error('oops', err)); * ``` * *
* * @example **Create a dataset using Arrow** * ```javascript * import { tableFromArrays, tableToIPC, Table } from 'apache-arrow'; * import { EdgeFile } from '@graphistry/node-api'; * * //columnar data is fastest; column per attribute; reuse across datasets * const edgesJSON = {'s': ['a1', 'b2'], 'd': ['b2', 'c3']}; * const edgesTable: Table = tableFromArrays(edgesJSON); * const edgesUint8: Uint8Array = tableToIPC(edgesArr, 'file'); * const edgesFile = new EdgeFile(edgesUint8, 'arrow'); * ``` * *
* * @example **Add files after the Dataset is instantiated but before it has been uploaded** * ```javascript * import { Dataset } from '@graphistry/node-api'; * const dataset = new Dataset( * { * node_encodings: { bindings: { 'node': 'n' } }, * edge_encodings: { bindings: { 'source': 's', 'destination': 'd' } }, * metadata: {}, * name: 'testdata', * } * ); * dataset.addFile(nodesFile); * dataset.addFile(edgesFile); * await dataset.upload(client); * console.log(`Dataset ${dataset.datasetID} uploaded to ${dataset.datasetURL}`); * ``` * *
* * @example **Set privacy on uploaded dataset** * ```javascript * import { Dataset, Privacy } from '@graphistry/node-api'; * const dataset = new Dataset( * { * node_encodings: { bindings: { } }, * edge_encodings: { bindings: { 'source': 's', 'destination': 'd' } }, * metadata: {}, * name: 'testdata', * } * ); * dataset.addFile(edgesFile); * await dataset.upload(client); * await dataset.setPrivacy(client); // see additional options below * ``` * *
* * @example **Set simple data-driven bindings for titles, colors, icons, and labels** * ```javascript * import { Dataset } from '@graphistry/node-api'; * const dataset = new Dataset( * { * * // See also simple title, color, ... and complex color, size, icon, badge, ... * // https://hub.graphistry.com/docs/api/2/rest/upload/colors * // https://hub.graphistry.com/docs/api/2/rest/upload/complex/ * node_encodings: { * * bindings: { * 'node': 'n', //id * 'node_title': 'some_title_column' //optional * }, * * complex: { * default: { * pointSizeEncoding: { * graphType: 'point', * encodingType: 'size', * attribute: 'payment', * variation: 'categorical', * mapping: { * fixed: { * big: 500, * normal: 200, * tiny: 10 * }, * other: 100 * } * } * } * } * }, * * // See also simple title, color, ... and complex color, size, icon, badge, ... * // https://hub.graphistry.com/docs/api/2/rest/upload/colors * // https://hub.graphistry.com/docs/api/2/rest/upload/complex/ * edge_encodings: { * * bindings: { * 'source': 's', 'destination': 'd' * }, * * complex: { * default: { * edgeColorEncoding: { * graphType: 'point', * encodingType: 'color', * attribute: 'time', * variation: 'continuous', * colors: ['blue', 'yellow', 'red'] * } * } * } * }, * * // Set brand & theme: Background, foreground, logo, page metadata * // https://hub.graphistry.com/docs/api/2/rest/upload/metadata/ * metadata: { * bg: { * color: 'silver' * }, * "logo": { * url: "http://a.com/logo.png", * } * }, * name: 'testdata', * }, * * nodesFile, * edgesFile * * // Visual and layout settings * // https://hub.graphistry.com/docs/api/1/rest/url/#urloptions * { * strongGravity: true, * edgeCurvature: 0.5 * } * * ); * await dataset.upload(); * ``` */ export class Dataset { //////////////////////////////////////////////////////////////////////////////// // Set at start or via managed APIs public readonly edgeFiles: EdgeFile[]; public readonly nodeFiles: NodeFile[]; public readonly bindings: Record; public readonly urlOpts: Record; // Set after upload private _datasetID?: string; public get datasetID(): string | undefined { return this._datasetID; } private _createDatasetResponse: any; public getCreateDatasetResponse(): any { return this._createDatasetResponse; } //////////////////////////////////////////////////////////////////////////////// private _usedClientProtocolHostname?: string; public get datasetURL(): string { if (!this._datasetID) { throw new Error('No dataset ID yet'); } if (!this._usedClientProtocolHostname) { throw new Error('No client protocol hostname yet'); } const xtra = Object.keys(this.urlOpts).length ? `&${Object.keys(this.urlOpts).map(k => `${k}=${this.urlOpts[k]}`).join('&')}` : ''; return `${this._usedClientProtocolHostname}/graph/graph.html?dataset=${this._datasetID}${xtra}`; } //////////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Dataset definitions including required node_encodings, edge_encodings, metadata and name. * Optional definitions include edge_hypergraph_transform, and description, and various subfields. * URL settings may also be specified for additional styling. * This method autopopulates definitions edge_files, and if provided, node_files. * * Node files are optional: Nodes will be synthesized based on edges if not provided. * * If files have not been uploaded yet, this method will upload them for you. * * For more information about bindings, see https://hub.graphistry.com/docs/api/2/rest/upload/ * * For more information about URL style settings, see https://hub.graphistry.com/docs/api/1/rest/url/#urloptions * * For more information about theming, see https://hub.graphistry.com/docs/api/2/rest/upload/metadata/ * * For more information on simple encodings, see https://hub.graphistry.com/docs/api/2/rest/upload/colors * * For more information on complex encodings, see https://hub.graphistry.com/docs/api/2/rest/upload/complex/ * * @param bindings JSON dictionary of bindings * @param nodeFiles File object(s) * @param edgeFiles File object(s) * @param urlOpts JSON dictionary of URL options */ constructor( bindings: Record = {}, edgeFiles: EdgeFile[] | EdgeFile = [], nodeFiles: NodeFile[] | NodeFile = [], urlOpts: Record = {} ) { this.bindings = bindings; this.edgeFiles = edgeFiles instanceof EdgeFile ? [edgeFiles] : edgeFiles; this.nodeFiles = nodeFiles instanceof NodeFile ? [nodeFiles] : nodeFiles; this.urlOpts = urlOpts; console.debug('Dataset constructor', this, { bindings, edgeFiles, nodeFiles, urlOpts }); for (const edgeFile of this.edgeFiles) { if (!edgeFile.fileFormat) { throw new Error('Edge file must have a fileType'); } } for (const nodeFile of this.nodeFiles) { if (!nodeFile.fileFormat) { throw new Error('Node file must have a fileType'); } } } //////////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Upload the dataset to the Graphistry server. * * If files have not been uploaded yet, this method will upload them for you. * * Upon completion, attributes datasetID and datasetURL will be set, as well as * createDatasetResponse and uploadResponse. * * @param client Client object * @returns Promise that resolves when the dataset is uploaded */ public async upload(client: ClientType): Promise { if (!client) { throw new Error('No client provided'); } if (!client.authTokenValid() && !client.isServerConfigured()) { throw new Error('Client is not configured, set token or creds'); } console.debug('Uploading dataset', {nodeFiles: this.nodeFiles, edgeFiles: this.edgeFiles}); //create+upload files as needed await Promise.all( this.nodeFiles.concat(this.edgeFiles).map(async (file) => { console.debug('Uploading file', file); const response = await file.createFile(client); console.debug(`Uploaded ${file.fileFormat} file`); if (!response) { throw new Error('File creation failed 1'); } const ok = await file.uploadData(client); if (!ok) { throw new Error('File upload failed 2'); } return file; })); //upload dataset and get its ID const fileBindings = { node_files: this.nodeFiles.map((file) => file.fileID), edge_files: this.edgeFiles.map((file) => file.fileID), }; const bindings = { ...(client.org ? { org_name: client.org } : {}), ...fileBindings, ...this.bindings }; await this.createDataset(client, bindings); return this; } //////////////////////////////////////////////////////////////////////////////// private async createDataset(client: ClientType, bindings: Record): Promise { this.fillMetadata(bindings, client); const dataJsonResults = await client.post('api/v2/upload/datasets/', bindings); this._createDatasetResponse = dataJsonResults; const datasetID = dataJsonResults.data.dataset_id; this._datasetID = datasetID; this._usedClientProtocolHostname = client.clientProtocolHostname; if (!datasetID) { throw new Error('Unexpected dataset response, check dataset._createDatasetResponse'); } return datasetID; } /** * Add one or more bindings to the existing ones. In case of conflicts, override the existing ones. * * For more information about each, see https://hub.graphistry.com/docs/api/2/rest/upload/ * * @param bindings JSON dictionary of bindings to be added to the existing ones */ public updateBindings(bindings: Record): void { for (const [key, value] of Object.entries(bindings)) { this.bindings[key] = value; } } /** * * See examples at top of file * * Add an additional node or edge file to the existing ones. * * @param file File object. Does not need to be uploaded yet. * **/ public addFile(file: File): void { console.debug('Adding file', file); if (file.type === FileType.Node) { this.nodeFiles.push(file); } else if (file.type === FileType.Edge) { this.edgeFiles.push(file); } else { throw new Error('Invalid File Type'); } } /////////////////////////////////////////////////////////////////////////////// private fillMetadata(data: any, client: ClientType): void{ if (!data) { throw new Error('No data to fill metadata; call setData() first or provide to File constructor'); } if (!data['metadata']) { data['metadata'] = {}; } const metadata = data['metadata']; if (!metadata['agent']) { metadata['agent'] = client.agent; } if (!metadata['agentversion']) { metadata['agentversion'] = client.version; } if (!metadata['apiversion']) { metadata['apiversion'] = '3'; } } ///////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Set the privacy mode of the dataset. All but the client are optional. * * @param client Client object * @param mode Privacy mode. One of 'private', 'public', 'organization' * @param modeAction Capability allowed when shared * @param invitedUsers List of user IDs to share with * @param notify Whether to notify users of the share * @param message Message to include in the notification * * * @returns Promise that resolves when the privacy is set * @throws Error if the dataset has not been uploaded yet * @throws Error if server call fails */ public async privacy( client: ClientType, mode: Mode = 'private', modeAction?: ModeAction, invitedUsers: string[] = [], notify = false, message = '') { if (!this._datasetID) { throw new Error('Dataset must be uploaded before setting privacy'); } const p = new Privacy( this._datasetID, 'dataset', mode, modeAction, invitedUsers, notify, message ); return await p.upload(client); } }