// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import { WebNNBackend } from '../backend-webnn'; import { LOG_DEBUG } from '../log'; // WebNN API currently does not have a TypeScript definition file. This file is a workaround with types generated from // WebNN API specification. // https://github.com/webmachinelearning/webnn/issues/677 /// export type TensorId = number; /** * Manages TensorId to MLTensor mapping. */ export interface TensorManager { /** * Reserve a new TensorId. */ reserveTensorId(): TensorId; /** * Release a TensorId. */ releaseTensorId(tensorId: TensorId): void; /** * Ensure a MLTensor is created for the TensorId. */ ensureTensor( tensorId: TensorId, dataType: MLOperandDataType, shape: readonly number[], copyOld: boolean, ): Promise; /** * Upload data to a MLTensor. */ upload(tensorId: TensorId, data: Uint8Array): void; /** * Download data from a MLTensor. */ download(tensorId: TensorId): Promise; download(tensorId: TensorId, dstTensor: ArrayBufferView | ArrayBuffer): Promise; /** * Release all tensors for a MLContext. */ releaseTensorsForContext(mlContext: MLContext): void; /** * Register an externally created MLTensor with a given MLContext and return a TensorId. */ registerTensor(mlContext: MLContext, mlTensor: MLTensor, dataType: MLOperandDataType, shape: number[]): TensorId; } let tensorGuid = 1; const createNewTensorId = (): TensorId => tensorGuid++; export type MLTensorEntry = [MLTensor, MLOperandDataType, readonly number[]]; /** * TensorTracker tracks the MLTensor and pending upload data. * * We need to track the MLTensor and pending upload data because we delay the creation of MLTensor until * we know the data type and shape. This is because future implementations of WebNN will only support creating * MLTensors with dataTypes and shape. */ class TensorTracker { private tensorEntry?: MLTensorEntry; private activeUpload?: Uint8Array; private tensorCache: MLTensorEntry[]; constructor( private mlContext?: MLContext, tensorEntry?: MLTensorEntry, ) { this.tensorEntry = tensorEntry; this.tensorCache = tensorEntry ? [tensorEntry] : []; } public get tensor(): MLTensor | undefined { return this.tensorEntry?.[0]; } public get context(): MLContext { if (!this.mlContext) { throw new Error('MLContext has not been set.'); } return this.mlContext; } public set context(mlContext: MLContext) { if (this.mlContext && this.mlContext !== mlContext) { throw new Error('MLTensor in use in a different MLContext.'); } this.mlContext = mlContext; } public destroy(): void { for (const [mlTensor] of this.tensorCache) { mlTensor.destroy(); } this.tensorCache = []; this.tensorEntry = undefined; } public trySelectTensor(context: MLContext, tryMLTensor: MLTensor): boolean { for (const [mlTensor, dataType, shape] of this.tensorCache) { if (tryMLTensor === mlTensor) { if (this.context !== context) { throw new Error('MLTensor cannot be registered with a different MLContext.'); } this.tensorEntry = [mlTensor, dataType, shape]; return true; } } return false; } public async ensureTensor( dataType: MLOperandDataType, shape: readonly number[], copyOld: boolean, ): Promise { if (this.tensorEntry) { const [mlTensor, existingDataType, existingShape] = this.tensorEntry; if (existingDataType === dataType && existingShape.every((v, i) => v === shape[i])) { return mlTensor; } } for (const [mlTensor, existingDataType, existingShape] of this.tensorCache) { if (existingDataType === dataType && existingShape.every((v, i) => v === shape[i])) { if (copyOld && this.tensorEntry) { // WebNN does not support copyTensorToTensor, so we need to read and write the tensors. LOG_DEBUG( 'verbose', () => `[WebNN] Slowdown may occur, having to copy existing tensor {dataType: ${dataType}, shape: ${shape}}`, ); const data = await this.context.readTensor(this.tensorEntry[0]); this.context.writeTensor(mlTensor, data); } this.tensorEntry = [mlTensor, existingDataType, existingShape]; return mlTensor; } } LOG_DEBUG('verbose', () => `[WebNN] MLContext.createTensor {dataType: ${dataType}, shape: ${shape}}`); // eslint-disable-next-line no-bitwise const usage = MLTensorUsage.READ | MLTensorUsage.WRITE; const tensor = await this.context.createTensor({ dataType, shape, // Assign both shape and dimensions while transitioning to new API. dimensions: shape, usage, }); this.tensorEntry = [tensor, dataType, shape]; this.tensorCache.push(this.tensorEntry); if (this.activeUpload) { this.mlContext?.writeTensor(tensor, this.activeUpload); this.activeUpload = undefined; } return tensor; } public upload(data: Uint8Array): void { if (!this.tensorEntry) { this.activeUpload = new Uint8Array(data); return; } this.mlContext?.writeTensor(this.tensorEntry[0], data); } public async download(dstBuffer?: ArrayBufferView | ArrayBuffer): Promise { if (this.activeUpload) { if (dstBuffer) { if (dstBuffer instanceof ArrayBuffer) { new Uint8Array(dstBuffer).set(this.activeUpload); } else { new Uint8Array(dstBuffer.buffer, dstBuffer.byteOffset, dstBuffer.byteLength).set(this.activeUpload); } return; } else { return this.activeUpload.buffer; } } if (!this.tensorEntry) { throw new Error('Tensor has not been created.'); } if (dstBuffer) { return this.context.readTensor(this.tensorEntry[0], dstBuffer); } return this.context.readTensor(this.tensorEntry[0]); } } class TensorManagerImpl implements TensorManager { private tensorsById = new Map(); private tensorIdsByContext = new Map>(); constructor(private backend: WebNNBackend) {} public reserveTensorId(): TensorId { const tensorId = createNewTensorId(); this.tensorsById.set(tensorId, new TensorTracker()); return tensorId; } public releaseTensorId(tensorId: TensorId): void { const tensorTracker = this.tensorsById.get(tensorId); if (!tensorTracker) { return; } tensorTracker.destroy(); this.tensorsById.delete(tensorId); for (const [mlContext, tensors] of this.tensorIdsByContext) { if (tensors.has(tensorId)) { tensors.delete(tensorId); if (tensors.size === 0) { this.tensorIdsByContext.delete(mlContext); } break; } } } public async ensureTensor( tensorId: TensorId, dataType: MLOperandDataType, shape: number[], copyOld: boolean, ): Promise { LOG_DEBUG( 'verbose', () => `[WebNN] TensorManager.ensureTensor {tensorId: ${tensorId}, dataType: ${ dataType }, shape: ${shape}, copyOld: ${copyOld}}`, ); const tensor = this.tensorsById.get(tensorId); if (!tensor) { throw new Error('Tensor not found.'); } tensor.context = this.backend.currentContext; if (!this.tensorIdsByContext.has(this.backend.currentContext)) { this.tensorIdsByContext.set(this.backend.currentContext, new Set()); } this.tensorIdsByContext.get(this.backend.currentContext)?.add(tensorId); return tensor.ensureTensor(dataType, shape, copyOld); } public upload(tensorId: TensorId, data: Uint8Array): void { this.tensorsById.get(tensorId)!.upload(data); } public async download(tensorId: TensorId): Promise; public async download(tensorId: TensorId, dstBuffer: ArrayBufferView | ArrayBuffer): Promise; async download(tensorId: TensorId, dstBuffer?: ArrayBufferView | ArrayBuffer): Promise { LOG_DEBUG( 'verbose', () => `[WebNN] TensorManager.download {tensorId: ${tensorId}, dstBuffer: ${dstBuffer?.byteLength}}`, ); return this.tensorsById.get(tensorId)!.download(dstBuffer); } public releaseTensorsForContext(mlContext: MLContext): void { const tensors = this.tensorIdsByContext.get(mlContext); if (!tensors) { return; } for (const tensorId of tensors) { this.tensorsById.get(tensorId)!.destroy(); this.tensorsById.delete(tensorId); } this.tensorIdsByContext.delete(mlContext); } public registerTensor( mlContext: MLContext, mlTensor: MLTensor, dataType: MLOperandDataType, shape: readonly number[], ): TensorId { for (const [tensorId, tensorTracker] of this.tensorsById) { if (tensorTracker.trySelectTensor(mlContext, mlTensor)) { return tensorId; } } const tensorId = createNewTensorId(); this.tensorsById.set(tensorId, new TensorTracker(mlContext, [mlTensor, dataType, shape])); let tensors = this.tensorIdsByContext.get(mlContext); if (!tensors) { tensors = new Set(); this.tensorIdsByContext.set(mlContext, tensors); } tensors.add(tensorId); return tensorId; } } export const createTensorManager = (...args: ConstructorParameters): TensorManager => new TensorManagerImpl(...args);