/** * WebCodecsDepthPipeline — WebCodecs → depth inference for real-time video. * * **Paths** * - **Default:** `VideoFrame` → `createImageBitmap` → `OffscreenCanvas` → `getImageData` * → `DepthEstimationService` (Transformers.js). One CPU readback is required for ONNX/WebGPU ML input today. * - **Optional WebGPU upload:** when `gpuDevice` is set and the frame **already matches** the target * width/height (no downscale), uses `GPUQueue.copyExternalImageToTexture` from the `VideoFrame` * (see https://www.w3.org/TR/webgpu/#dom-gpuqueue-copyexternalimagetotexture ) then a single * `copyTextureToBuffer` readback into `ImageData`. This drops the `ImageBitmap` + 2D canvas blit * for that case — a step toward a full GPU-resident depth stack once the model consumes `GPUTexture`. * * Pipeline (default): VideoDecoder → VideoFrame → bitmap/canvas → DepthEstimationService * * @see W.157: WebCodecs decode path avoids MediaSource + canvas drawImage for decode * @see G.151.01: Decouple video playback rate from VR render rate */ import type { DepthResult } from './DepthEstimationService'; export interface WebCodecsDepthConfig { /** Maximum frames to process per second. Default: 30 */ maxFps: number; /** Maximum resolution for depth inference. Default: 512 */ maxDepthResolution: number; /** Temporal smoothing alpha. Default: 0.8 */ temporalAlpha: number; /** Codec to accept. Default: 'vp9' */ codec: 'h264' | 'vp9' | 'av1'; /** * Optional WebGPU device. When set and the decoded frame size already equals the * inference size (no downscale), uploads with copyExternalImageToTexture instead of * createImageBitmap + 2D canvas. */ gpuDevice?: GPUDevice; /** Frame callback — called with each depth result */ onFrame?: (result: DepthResult, frameIndex: number, timestamp: number) => void; /** Error callback */ onError?: (error: Error) => void; } export interface WebCodecsDepthStats { /** Total frames decoded */ framesDecoded: number; /** Total frames processed for depth */ framesProcessed: number; /** Frames skipped (rate limiting) */ framesSkipped: number; /** Average decode time in ms */ avgDecodeMs: number; /** Average depth inference time in ms */ avgInferenceMs: number; /** Pipeline running state */ running: boolean; } /** WebGPU row pitch for copyTextureToBuffer (256-byte aligned). */ export declare function webgpuBytesPerRowRgba8(widthPx: number): number; /** * VideoFrame → rgba8unorm texture via copyExternalImageToTexture, then one readback to ImageData. * Caller must close `frame` after this resolves (this does not close the frame). */ export declare function videoFrameToImageDataViaWebGPU(device: GPUDevice, frame: VideoFrame, width: number, height: number): Promise; /** * WebCodecs video depth pipeline (decode → pixels → DepthEstimationService). * * Uses GPU-accelerated **upload** when `gpuDevice` is provided and no resize is needed; * Transformers.js depth still receives `ImageData` until a texture-native path exists. * * Usage: * ```typescript * const pipeline = new WebCodecsDepthPipeline(); * await pipeline.initialize({ onFrame: (depth, idx) => applyDisplacement(depth) }); * pipeline.feedChunk(encodedVideoChunk); * // ... later * pipeline.dispose(); * ``` */ export declare class WebCodecsDepthPipeline { private decoder; private config; private depthService; private gpuDevice; private canvas; private ctx; private _running; private _disposed; private lastProcessTime; private minFrameInterval; private _framesDecoded; private _framesProcessed; private _framesSkipped; private _totalDecodeMs; private _totalInferenceMs; constructor(config?: Partial); /** * Check if WebCodecs API is available in the current environment. */ static isSupported(): boolean; /** * Initialize the pipeline: set up VideoDecoder and DepthEstimationService. */ initialize(config?: Partial): Promise; /** * Feed an encoded video chunk to the decoder. * The chunk will be decoded and processed for depth asynchronously. */ feedChunk(chunk: any): void; /** * Process a raw VideoFrame directly (for cases where decoding is handled externally). */ processFrame(frame: any): Promise; /** * Get pipeline statistics. */ get stats(): WebCodecsDepthStats; /** * Handle a decoded VideoFrame: rate-limit, extract pixels, run depth inference. */ private _handleDecodedFrame; /** * Flush any remaining frames in the decoder. */ flush(): Promise; /** * Stop the pipeline and release all resources. */ dispose(): void; } //# sourceMappingURL=WebCodecsDepthPipeline.d.ts.map