/* eslint-disable @typescript-eslint/no-explicit-any */ import { ClientType } from './types.js'; /** * @internal * *
* * ## FileType * * Enumeration used by {@link File} subclasses {@link EdgeFile} and {@link NodeFile} to specify whether the file is a node or edge file. * * Primarily useful for TypeScript and runtime checks to avoid mixups when passing {@link File} objects to {@link Dataset} * */ export enum FileType { Node, Edge } //////////////////////////////////////////////////////////////////////////////// /** * # File examples * * {@link File} objects are used for uploading data and then reusing as part of {@link Dataset} graph visualizations * * Powerfully, the same file may be reused in multiple {@link Dataset}s, so many variants can be made cheaply and quickly. * * For configuring supported file formats, see https://hub.graphistry.com/docs/api/2/rest/files/ . * *
* * --- * *
* * @example **Upload an {@link EdgeFile} from a JSON object in a columnar form** * ```javascript * import { EdgeFile } from '@graphistry/node-api'; * const edgesFile = new EdgeFile( * { * 's': ['a', 'b', 'c'], * 'd': ['d', 'e', 'f'], * 'v': ['v1', 'v2', 'v3'] * } * ); * await edgesFile.upload(client); * console.log(`EdgeFile uploaded as ID ${edgesFile.fileID}`); * ``` * *
* * @example **Upload an {@link EdgeFile} from a JSON object in a row-oriented form** * ```javascript * import { EdgeFile } from '@graphistry/node-api'; * const edgesFile = new EdgeFile( * [ * {'s': 'a', 'd': 'd', 'v': 'v1'}, * {'s': 'b', 'd': 'e', 'v': 'v2'}, * {'s': 'c', 'd': 'f', 'v': 'v3'} * ], * 'json', * 'my nodes file', * * // JSON parsing options: * // - https://hub.graphistry.com/docs/api/2/rest/upload/data/#uploadjson2 * // - https://pandas.pydata.org/docs/reference/api/pandas.read_json.html * // * // Also: file_compression, sql_transforms, ... * // https://hub.graphistry.com/docs/api/2/rest/files/ * {parser_options: {orient: 'records'}} * * ); * await edgesFile.upload(client); * console.log(`EdgeFile uploaded as ID ${edgesFile.fileID}`); * ``` * *
* * @example **Upload an {@link EdgeFile} using promises** * ```javascript * import { EdgeFile } from '@graphistry/node-api'; * const edgesFile = new EdgeFile({'s': ['a', 'b', 'c'], 'd': ['d', 'e', 'f'], 'v': ['v1', 'v2', 'v3']}); * edgesFile.upload(client).then( * () => console.log(`EdgeFile uploaded as ID ${edgesFile.fileID}`) * ).catch(err => console.error('oops', err)); * ``` * *
* * @example **Upload a {@link NodeFile} from a JSON object** * ```javascript * import { NodeFile } from '@graphistry/node-api'; * const nodesFile = new NodeFile({'n': ['a', 'b', 'c']}); * await nodesFile.upload(client); * console.log(`NodeFile uploaded as ID ${nodesFile.fileID}`); * ``` * *
* * @example **Upload a {@link NodeFile} from an Apache Arrow Table** * ```javascript * import { tableFromArrays, tableToIPC } from 'apache-arrow'; * import { NodeFile } from '@graphistry/node-api'; * const json = {'n': ['a', 'b', 'c']}; * const arr = tableFromArrays(json); * const uint8Buf = tableToIPC(arr); * const nodesFile = new NodeFile(uint8Buf, 'arrow'); * await nodesFile.upload(client); * console.log(`NodeFile uploaded as ID ${nodesFile.fileID}`); * ``` * *
* * @example **Create a {@link File} by ID (e.g., previously uploaded) for use with {@link Dataset}s** * ```javascript * import { EdgeFile, Dataset } from '@graphistry/node-api'; * const edgesFile = new EdgeFile('my_file_id'); * await (new Dataset(bindings, edgesFile)).upload(client); * console.log(`Dataset uploaded as ID ${dataset.datasetID}`); * ``` */ export class File { //////////////////////////////////////////////////////////////////////////////// // Late-bindable by user private _data: any; public get data(): any | undefined { return this._data; } // Late-bound during uploads private _fileID?: string; public get fileID(): string | undefined { return this._fileID; } private _fileCreated = false; public get fileCreated(): boolean { return this._fileCreated; } private _fileCreateResponse : any; public get fileCreateResponse(): any { return this._fileCreateResponse; } private _fileUploaded = false; public get fileUploaded(): any { return this._fileUploaded; } private _fileValidated = false; public get fileValidated(): any { return this._fileValidated; } private _fileUploadResponse : any; public get fileUploadResponse(): any { return this._fileUploadResponse; } // Bound at creation public readonly createOpts: Record; public readonly uploadUrlOpts: string; public readonly fileFormat: string; public readonly name: string; public readonly type: FileType; //////////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Create a new {@link File} object for uploading or use with {@link Dataset} * * For more information on the available options, see: * * Creation step metadata options: https://hub.graphistry.com/docs/api/2/rest/files/ * * Upload step options: https://hub.graphistry.com/docs/api/2/rest/files/#uploadfiledata * * @param type FileType.Node or FileType.Edge * @param data Payload to pass to node-fetch. Use TypedArrays such as Uint8Array for binary data such as arrow * @param fileFormat File format to use, e.g. 'json', 'csv', 'arrow', 'parquet', 'orc', 'xls' * @param name Name of the file to use, e.g. 'my-file' * @param createOpts JSON post body options to use in createFile() * @param uploadUrlOpts URL options to use in uploadData() */ constructor(type: FileType, data: any = undefined, fileFormat = 'json', name = 'my file', createOpts = {}, uploadUrlOpts = '') { if (typeof(data) == 'string') { this._fileID = data; this._fileCreated = true; this._fileUploaded = true; } else { this._data = data; } this.createOpts = createOpts; this.uploadUrlOpts = uploadUrlOpts; this.fileFormat = fileFormat; this.name = name; this.type = type; } //////////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Upload curent {@link File} object to the server * * By default, this will skip reuploading files that have already been uploaded. * * @param client {@link Client} object to use for uploading * @param force If true, will force upload even if file has already been uploaded * @returns Promise that resolves to the uploaded File object when it completes uploading */ public async upload(client : ClientType, force = false): Promise { if (!client) { throw new Error('No client provided'); } await this.createFile(client, force); if (!this._fileID) { throw new Error('Unexpected file creation response, check file._fileCreateResponse'); } await this.uploadData(client, force); if (!this._fileCreated) { throw new Error('Unexpected file upload response, check file._fileUploadResponse'); } if (!this._fileValidated) { throw new Error('Unexpected file validation response, check file._fileValidateResponse'); } return this; } //////////////////////////////////////////////////////////////////////////////// /** * * See examples at top of file * * Helper function to create the file on the server but not yet upload its data * * By default, this will skip recreating files that have already been created. * * @param client {@link Client} object to use for uploading * @param force If true, will force creation of a new ID even if file has already been uploaded * @returns */ public async createFile(client : ClientType, force = false): Promise { if (!force && this._fileCreated) { console.debug('File already created, skipping'); return this._fileCreated; } console.debug('Creating file'); const fileJsonResults = await client.post( 'api/v2/files/', { file_type: this.fileFormat, ...(client.org ? { org_name: client.org } : {}), ...this.createOpts }); console.debug('File creation response:', fileJsonResults); this._fileCreateResponse = fileJsonResults; this._fileID = fileJsonResults.file_id; this._fileCreated = !!fileJsonResults.file_id; return fileJsonResults; } /** * * See examples at top of file * * Helper function to upload the data to the server * * By default, this will skip reuploading data that has already been uploaded. * * @param client {@link Client} object to use for uploading * @param force If true, will force upload even if file has already been uploaded * @returns */ public async uploadData(client : ClientType, force = false): Promise { if (!force && this._fileUploaded) { return this._fileUploaded; } this.fillMetadata(this._data, client); const results = await client.post( `api/v2/upload/files/${this._fileID}${ this.uploadUrlOpts ? `?${this.uploadUrlOpts}` : ''}`, this._data, this.fileFormat != 'json' ? {} : undefined ); this._fileUploadResponse = results; this._fileUploaded = !!results.is_uploaded; this._fileValidated = !!results.is_valid; return results; } /** * * See examples at top of file * * Populate data for later uploading if it wasn't set during construction * * Cannot run this function if the file has already been uploaded * * Overwrites any existing data * * @param data Data to use for uploading */ public setData(data: any) { if (this._fileUploaded) { throw new Error('Cannot set data after file has been successfully uploaded'); } this._data = data; } /////////////////////////////////////////////////////////////////////////////// private fillMetadata(data: any, client: ClientType): void { if (!data) { throw new Error('No data to fill metadata; call setData() first or provide to File constructor'); } if (data instanceof Uint8Array) { return; } if (!data['agent_name']) { data['agent_name'] = client.agent; } if (!data['agent_version']) { data['agent_version'] = client.version; } } } //////////////////////////////////////////////////////////////////////////////// /** * Helper class for tracking intent when creating a {@link File} object for uploading * * See examples at top of file * */ export class EdgeFile extends File { constructor(data: any = undefined, fileFormat = 'json', name = 'my file', createOpts = {}, urlOpts = '') { super(FileType.Edge, data, fileFormat, name, createOpts, urlOpts); } } /** * Helper class for tracking intent when creating a {@link File} object for uploading * * See examples at top of file */ export class NodeFile extends File { constructor(data: any = undefined, fileFormat = 'json', name = 'my file', createOpts = {}, urlOpts = '') { super(FileType.Node, data, fileFormat, name, createOpts, urlOpts); } }