import type * as plugins from './plugins.js'; import * as interfaces from '../ts_interfaces/index.js'; const browserOperationTimeoutMs = 35_000; const browserTransportStopTimeoutMs = 1_000; const textEncoder = new TextEncoder(); interface IPendingBrowserOperation { resolve(resultArg: interfaces.TControllerBrowserOperationResult): void; reject(reasonArg: unknown): void; abortSignal?: AbortSignal; abortListener?: () => void; timeout: ReturnType; } export interface IControllerBrowserViewTransportClientOptions { streamGeneration: number; events: plugins.typedrequestInterfaces.TVirtualStream<'receive'>; operations: plugins.typedrequestInterfaces.TVirtualStream<'send'>; onEvent(eventArg: interfaces.TControllerBrowserViewEvent): void; onFailure(errorArg: unknown, recoverableArg: boolean): void; } export class ControllerBrowserOperationError extends Error { constructor( public readonly code: string, messageArg: string, ) { super(messageArg); this.name = 'ControllerBrowserOperationError'; } } type TObject = Record; const assertObject = (valueArg: unknown, nameArg: string): TObject => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new interfaces.ControllerBrowserTransportProtocolError(`${nameArg} must be an object.`); } const prototype = Object.getPrototypeOf(valueArg); if (prototype !== Object.prototype && prototype !== null) { throw new interfaces.ControllerBrowserTransportProtocolError(`${nameArg} must be a plain object.`); } return valueArg as TObject; }; const assertKeys = ( valueArg: TObject, requiredArg: readonly string[], optionalArg: readonly string[] = [], ): void => { const allowed = new Set([...requiredArg, ...optionalArg]); if ( !requiredArg.every((keyArg) => Object.prototype.hasOwnProperty.call(valueArg, keyArg)) || Object.keys(valueArg).some((keyArg) => !allowed.has(keyArg)) ) throw new interfaces.ControllerBrowserTransportProtocolError('Browser packet keys are invalid.'); }; const assertBoundedString = (valueArg: unknown, nameArg: string, maximumBytesArg = 512): string => { if ( typeof valueArg !== 'string' || valueArg.length === 0 || textEncoder.encode(valueArg).byteLength > maximumBytesArg ) throw new interfaces.ControllerBrowserTransportProtocolError(`${nameArg} is invalid.`); return valueArg; }; const assertSafeInteger = (valueArg: unknown, nameArg: string, minimumArg = 0): number => { if (!Number.isSafeInteger(valueArg) || (valueArg as number) < minimumArg) { throw new interfaces.ControllerBrowserTransportProtocolError(`${nameArg} is invalid.`); } return valueArg as number; }; const assertFiniteNumber = (valueArg: unknown, nameArg: string): number => { if (typeof valueArg !== 'number' || !Number.isFinite(valueArg)) { throw new interfaces.ControllerBrowserTransportProtocolError(`${nameArg} is invalid.`); } return valueArg; }; const assertBrowserViewport = (valueArg: unknown): interfaces.IControllerBrowserViewport => { const value = assertObject(valueArg, 'viewport'); assertKeys(value, ['width', 'height', 'deviceScaleFactor']); const width = assertSafeInteger(value.width, 'viewport.width', 1); const height = assertSafeInteger(value.height, 'viewport.height', 1); const deviceScaleFactor = assertFiniteNumber(value.deviceScaleFactor, 'viewport.deviceScaleFactor'); if ( width > 4096 || height > 4096 || deviceScaleFactor < 0.25 || deviceScaleFactor > 3 || Math.ceil(width * deviceScaleFactor) * Math.ceil(height * deviceScaleFactor) > 8_294_400 ) throw new interfaces.ControllerBrowserTransportProtocolError('viewport is out of bounds.'); return { width, height, deviceScaleFactor }; }; const assertBrowserError = ( valueArg: unknown, fatalRequiredArg = true, ): { code: string; message: string; fatal: boolean; tabId?: string } => { const value = assertObject(valueArg, 'browser error'); assertKeys(value, fatalRequiredArg ? ['code', 'message', 'fatal'] : ['code', 'message'], ['tabId']); if (fatalRequiredArg && typeof value.fatal !== 'boolean') { throw new interfaces.ControllerBrowserTransportProtocolError('browser error fatal flag is invalid.'); } const tabId = value.tabId === undefined ? undefined : assertBoundedString(value.tabId, 'browser error tabId'); return { code: assertBoundedString(value.code, 'browser error code', 128), message: assertBoundedString(value.message, 'browser error message', 512), fatal: fatalRequiredArg ? value.fatal as boolean : false, ...(tabId === undefined ? {} : { tabId }), }; }; const assertBrowserState = (valueArg: unknown): interfaces.IControllerBrowserViewState => { const value = assertObject(valueArg, 'browser state'); assertKeys( value, ['revision', 'status', 'activeTabId', 'viewportRevision', 'viewport', 'tabs'], ['lastError', 'videoAcceleration', 'videoSource'], ); if (!['stopped', 'starting', 'running', 'stopping'].includes(String(value.status))) { throw new interfaces.ControllerBrowserTransportProtocolError('browser state status is invalid.'); } if (value.activeTabId !== null && typeof value.activeTabId !== 'string') { throw new interfaces.ControllerBrowserTransportProtocolError('browser state activeTabId is invalid.'); } if (!Array.isArray(value.tabs) || value.tabs.length > 64) { throw new interfaces.ControllerBrowserTransportProtocolError('browser state tabs are invalid.'); } const tabs = value.tabs.map((tabArg): interfaces.IControllerBrowserTabState => { const tab = assertObject(tabArg, 'browser tab'); assertKeys(tab, [ 'id', 'url', 'title', 'active', 'status', 'generation', 'appliedViewportRevision', 'streaming', ], ['dialog']); if ( typeof tab.active !== 'boolean' || typeof tab.streaming !== 'boolean' || !['open', 'crashed'].includes(String(tab.status)) ) throw new interfaces.ControllerBrowserTransportProtocolError('browser tab state is invalid.'); let dialog: interfaces.IControllerBrowserTabState['dialog']; if (tab.dialog !== undefined) { const value = assertObject(tab.dialog, 'website dialog'); assertKeys(value, ['id', 'type', 'message', 'defaultPrompt', 'url']); if (!['alert', 'confirm', 'prompt', 'beforeunload'].includes(String(value.type)) || typeof value.message !== 'string' || value.message.length > 16384 || typeof value.defaultPrompt !== 'string' || value.defaultPrompt.length > 4096 || typeof value.url !== 'string' || value.url.length > 4096) { throw new interfaces.ControllerBrowserTransportProtocolError('Website dialog is invalid.'); } dialog = { id: assertBoundedString(value.id, 'dialog id'), type: value.type as NonNullable['type'], message: value.message, defaultPrompt: value.defaultPrompt, url: value.url }; } return { id: assertBoundedString(tab.id, 'browser tab id'), url: typeof tab.url === 'string' && textEncoder.encode(tab.url).byteLength <= 16 * 1024 ? tab.url : (() => { throw new interfaces.ControllerBrowserTransportProtocolError('browser tab URL is invalid.'); })(), title: typeof tab.title === 'string' && textEncoder.encode(tab.title).byteLength <= 4 * 1024 ? tab.title : (() => { throw new interfaces.ControllerBrowserTransportProtocolError('browser tab title is invalid.'); })(), active: tab.active, status: tab.status as 'open' | 'crashed', generation: assertSafeInteger(tab.generation, 'browser tab generation'), appliedViewportRevision: assertSafeInteger( tab.appliedViewportRevision, 'browser tab viewport revision', ), streaming: tab.streaming, ...(dialog ? { dialog } : {}), }; }); const lastError = value.lastError === undefined ? undefined : assertBrowserError(value.lastError); const videoSource = value.videoSource === undefined ? undefined : assertVideoSource(value.videoSource); const viewport = assertBrowserViewport(value.viewport); if (videoSource && (videoSource.tabId !== value.activeTabId || videoSource.generation !== tabs.find(tab => tab.id === value.activeTabId)?.generation || videoSource.viewportRevision !== value.viewportRevision || videoSource.viewport.width !== viewport.width || videoSource.viewport.height !== viewport.height || videoSource.viewport.deviceScaleFactor !== viewport.deviceScaleFactor)) { throw new interfaces.ControllerBrowserTransportProtocolError('Video source does not match browser state.'); } return { revision: assertSafeInteger(value.revision, 'browser state revision', 1), status: value.status as interfaces.IControllerBrowserViewState['status'], activeTabId: value.activeTabId as string | null, viewportRevision: assertSafeInteger(value.viewportRevision, 'browser state viewport revision'), viewport, tabs, ...(value.videoAcceleration === undefined ? {} : { videoAcceleration: assertVideoAcceleration(value.videoAcceleration) }), ...(videoSource === undefined ? {} : { videoSource }), ...(lastError === undefined ? {} : { lastError }), }; }; const assertVideoAcceleration = (valueArg: unknown): NonNullable => { const value = assertObject(valueArg, 'video acceleration'); assertKeys(value, ['renderer', 'compositing', 'videoEncoding']); return { renderer: assertBoundedString(value.renderer, 'GPU renderer', 2048), compositing: assertBoundedString(value.compositing, 'GPU compositing', 128), videoEncoding: assertBoundedString(value.videoEncoding, 'GPU video encoding', 128) }; }; const assertVideoSource = (valueArg: unknown): interfaces.IControllerBrowserVideoOffer['source'] => { const source = assertObject(valueArg, 'video source'); assertKeys(source, ['tabId', 'generation', 'viewportRevision', 'viewport'], ['frameAlignment', 'rtpTimestampFloor']); if ((source.frameAlignment !== undefined || source.rtpTimestampFloor !== undefined) && (source.frameAlignment !== 2 || !Number.isSafeInteger(source.rtpTimestampFloor) || (source.rtpTimestampFloor as number) < 0 || (source.rtpTimestampFloor as number) > 0xffffffff)) { throw new interfaces.ControllerBrowserTransportProtocolError('Native video presentation fence is invalid.'); } return { tabId: assertBoundedString(source.tabId, 'video tabId', 128), generation: assertSafeInteger(source.generation, 'video generation'), viewportRevision: assertSafeInteger(source.viewportRevision, 'video viewportRevision', 1), viewport: assertBrowserViewport(source.viewport), ...(source.frameAlignment === undefined ? {} : { frameAlignment: 2 as const, rtpTimestampFloor: source.rtpTimestampFloor as number }) }; }; const assertVideoOffer = (valueArg: unknown): interfaces.IControllerBrowserVideoOffer => { const value = assertObject(valueArg, 'video offer'); assertKeys(value, ['negotiationId', 'source', 'description', 'iceServers', 'iceTransportPolicy']); const source = assertVideoSource(value.source); const description = interfaces.assertControllerBrowserVideoDescription(value.description, 'offer'); if (!['all', 'relay'].includes(String(value.iceTransportPolicy)) || !Array.isArray(value.iceServers) || value.iceServers.length > 8) { throw new interfaces.ControllerBrowserTransportProtocolError('Video offer is invalid.'); } const iceServers = value.iceServers.map((entryArg) => { const entry = assertObject(entryArg, 'ICE server'); assertKeys(entry, ['urls'], ['username', 'credential']); const urls = Array.isArray(entry.urls) ? entry.urls : [entry.urls]; if (urls.length < 1 || urls.length > 8) throw new interfaces.ControllerBrowserTransportProtocolError('ICE URLs are invalid.'); const parsedUrls = urls.map((urlArg) => { const url = assertBoundedString(urlArg, 'ICE URL', 2048); if (!/^(stun|stuns|turn|turns):[^\s@]+$/.test(url)) throw new interfaces.ControllerBrowserTransportProtocolError('ICE URL is invalid.'); return url; }); for (const key of ['username', 'credential']) { if (entry[key] !== undefined && (typeof entry[key] !== 'string' || entry[key].length > 2048)) { throw new interfaces.ControllerBrowserTransportProtocolError('ICE credentials are invalid.'); } } return { urls: parsedUrls, ...(entry.username === undefined ? {} : { username: entry.username as string }), ...(entry.credential === undefined ? {} : { credential: entry.credential as string }) }; }); return { negotiationId: assertBoundedString(value.negotiationId, 'negotiationId', 128), source, description, iceServers, iceTransportPolicy: value.iceTransportPolicy as 'all' | 'relay', }; }; const assertVideoStatistics = (valueArg: unknown): interfaces.IControllerBrowserVideoStatistics => { const value = assertObject(valueArg, 'video statistics'); const numericKeys = ['frameWidth', 'frameHeight', 'framesPerSecond', 'roundTripTime', 'scaleResolutionDownBy', 'totalEncodeTime'] as const; const stringKeys = ['encoderImplementation', 'qualityLimitationReason', 'transportProtocol'] as const; assertKeys(value, ['connectionState', 'visible', 'framesEncoded', 'bytesSent'], [...numericKeys, ...stringKeys, 'powerEfficientEncoder']); if (typeof value.visible !== 'boolean' || (value.powerEfficientEncoder !== undefined && typeof value.powerEfficientEncoder !== 'boolean')) throw new interfaces.ControllerBrowserTransportProtocolError('Video statistics flags are invalid.'); const result: interfaces.IControllerBrowserVideoStatistics = { connectionState: assertBoundedString(value.connectionState, 'video connection state', 128), visible: value.visible, framesEncoded: assertSafeInteger(value.framesEncoded, 'framesEncoded'), bytesSent: assertSafeInteger(value.bytesSent, 'bytesSent'), ...(value.powerEfficientEncoder === undefined ? {} : { powerEfficientEncoder: value.powerEfficientEncoder }), }; for (const key of numericKeys) if (value[key] !== undefined) { const number = assertFiniteNumber(value[key], key); if (number < 0) throw new interfaces.ControllerBrowserTransportProtocolError('Video statistics are negative.'); result[key] = number; } for (const key of stringKeys) if (value[key] !== undefined) { if (typeof value[key] !== 'string' || textEncoder.encode(value[key]).byteLength > 512) { throw new interfaces.ControllerBrowserTransportProtocolError('Video statistics text is invalid.'); } result[key] = value[key]; } return result; }; const assertServerPacket = (valueArg: unknown): interfaces.TControllerBrowserServerPacket => { const value = assertObject(valueArg, 'browser server packet'); if (value.protocol !== interfaces.controllerBrowserTransportProtocol) { throw new interfaces.ControllerBrowserTransportProtocolError('browser packet protocol is invalid.'); } const streamGeneration = assertSafeInteger(value.streamGeneration, 'streamGeneration', 1); if (value.type === 'state') { assertKeys(value, ['protocol', 'streamGeneration', 'type', 'state']); return { protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration, type: 'state', state: assertBrowserState(value.state), }; } if (value.type === 'error') { assertKeys(value, ['protocol', 'streamGeneration', 'type', 'error']); return { protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration, type: 'error', error: assertBrowserError(value.error), }; } if (value.type === 'operationResult') { assertKeys(value, ['protocol', 'streamGeneration', 'type', 'operationId', 'result']); const result = assertObject(value.result, 'browser operation result'); if (result.status === 'succeeded') { assertKeys(result, ['status'], ['state', 'viewportResult', 'videoOffer', 'videoStatistics']); let viewportResult: interfaces.IControllerBrowserViewportResult | undefined; if (result.viewportResult !== undefined) { const acceptedViewport = assertObject(result.viewportResult, 'viewportResult'); assertKeys(acceptedViewport, ['viewport', 'viewportRevision']); viewportResult = { viewport: assertBrowserViewport(acceptedViewport.viewport), viewportRevision: assertSafeInteger(acceptedViewport.viewportRevision, 'viewportRevision'), }; } return { protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration, type: 'operationResult', operationId: assertSafeInteger(value.operationId, 'operationId', 1), result: { status: 'succeeded', ...(result.state === undefined ? {} : { state: assertBrowserState(result.state) }), ...(viewportResult === undefined ? {} : { viewportResult }), ...(result.videoOffer === undefined ? {} : { videoOffer: assertVideoOffer(result.videoOffer) }), ...(result.videoStatistics === undefined ? {} : { videoStatistics: assertVideoStatistics(result.videoStatistics) }), }, }; } if (result.status === 'failed') { assertKeys(result, ['status', 'error']); const error = assertBrowserError(result.error, false); return { protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration, type: 'operationResult', operationId: assertSafeInteger(value.operationId, 'operationId', 1), result: { status: 'failed', error: { code: error.code, message: error.message } }, }; } if (result.status === 'cancelled' || result.status === 'superseded') { assertKeys(result, ['status']); return { protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration, type: 'operationResult', operationId: assertSafeInteger(value.operationId, 'operationId', 1), result: { status: result.status }, }; } } throw new interfaces.ControllerBrowserTransportProtocolError('browser server packet type is invalid.'); }; export class ControllerBrowserViewTransportClient { private lifecycle: 'new' | 'opening' | 'active' | 'failed' | 'stopped' = 'new'; private nextOperationId = 0; private readonly pendingOperations = new Map(); private startTask?: Promise; private receiveTask?: Promise; private stopTask?: Promise; private failureReported = false; constructor(private readonly options: IControllerBrowserViewTransportClientOptions) {} public get streamGeneration(): number { return this.options.streamGeneration; } public start(): Promise { if (this.lifecycle === 'active') return Promise.resolve(); if (this.startTask) return this.startTask; if (this.lifecycle !== 'new') { return Promise.reject(new Error('The browser transport cannot be started.')); } this.lifecycle = 'opening'; const receiveTask = this.receiveEvents(); this.receiveTask = receiveTask; void receiveTask.catch((errorArg) => this.fail( errorArg, !(errorArg instanceof interfaces.ControllerBrowserTransportProtocolError), )); const startTask = Promise.all([ this.options.events.opened, this.options.operations.opened, ]).then(() => { if (this.lifecycle !== 'opening') throw new Error('The browser transport stopped while opening.'); this.lifecycle = 'active'; }).catch((errorArg) => { this.fail(errorArg, true); throw errorArg; }); this.startTask = startTask; return startTask; } public async operate( operationArg: interfaces.TControllerBrowserViewOperation, signalArg?: AbortSignal, ): Promise { if (this.lifecycle !== 'active') throw new Error('The browser transport is unavailable.'); signalArg?.throwIfAborted(); if ( this.pendingOperations.size >= interfaces.controllerBrowserTransportMaximumPendingOperations ) throw new Error('The browser operation capacity was reached.'); const operationId = ++this.nextOperationId; const packet = interfaces.encodeControllerBrowserTransportPacket({ protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration: this.options.streamGeneration, type: 'operation', operationId, operation: operationArg, }); const result = new Promise((resolve, reject) => { const timeout = setTimeout(() => { this.cancelPendingOperation( operationId, new DOMException('The browser operation timed out.', 'TimeoutError'), ); }, browserOperationTimeoutMs); const pending: IPendingBrowserOperation = { resolve, reject, timeout }; if (signalArg) { pending.abortSignal = signalArg; pending.abortListener = () => this.cancelPendingOperation( operationId, signalArg.reason ?? new DOMException('The operation was aborted.', 'AbortError'), ); signalArg.addEventListener('abort', pending.abortListener, { once: true }); } this.pendingOperations.set(operationId, pending); }); let sendTask: Promise; try { sendTask = this.options.operations.send(packet); } catch (errorArg) { this.rejectPendingOperation(operationId, errorArg); this.fail(errorArg, true); return result; } void sendTask.catch((errorArg) => { this.rejectPendingOperation(operationId, errorArg); this.fail(errorArg, true); }); return result; } public stop(reasonArg: unknown = new Error('The browser transport stopped.')): Promise { if (this.stopTask) return this.stopTask; this.lifecycle = 'stopped'; this.rejectAllPending(reasonArg); let timeout: ReturnType | undefined; const settlement = Promise.allSettled([ Promise.resolve().then(() => this.options.events.abort(reasonArg)), Promise.resolve().then(() => this.options.operations.abort(reasonArg)), this.startTask, this.receiveTask, this.options.events.closed, this.options.operations.closed, ].filter((candidateArg): candidateArg is Promise => candidateArg !== undefined)) .then(() => undefined); const boundedStop = Promise.race([ settlement, new Promise((resolve) => { timeout = setTimeout(resolve, browserTransportStopTimeoutMs); }), ]).finally(() => { if (timeout) clearTimeout(timeout); }); this.stopTask = boundedStop; return boundedStop; } private async receiveEvents(): Promise { while (this.lifecycle !== 'failed' && this.lifecycle !== 'stopped') { const chunk = await this.options.events.receive(); if (chunk === undefined) { if (this.isStopped()) return; await this.options.events.accept(); throw new Error('The browser event stream closed unexpectedly.'); } const packet = assertServerPacket( interfaces.decodeControllerBrowserTransportPacket(chunk), ); if (packet.streamGeneration !== this.options.streamGeneration) { throw new interfaces.ControllerBrowserTransportProtocolError( 'The browser packet belongs to a stale stream generation.', ); } if (packet.type === 'operationResult') { this.settlePendingOperation(packet.operationId, packet.result); continue; } if (packet.type === 'state') { this.options.onEvent({ type: 'state', state: packet.state }); continue; } if (packet.type === 'error') { this.options.onEvent({ type: 'error', error: packet.error }); if (packet.error.fatal) { this.fail(new ControllerBrowserOperationError(packet.error.code, packet.error.message), false); return; } continue; } } } private cancelPendingOperation(operationIdArg: number, reasonArg: unknown): void { if (!this.rejectPendingOperation(operationIdArg, reasonArg)) return; const packet = interfaces.encodeControllerBrowserTransportPacket({ protocol: interfaces.controllerBrowserTransportProtocol, streamGeneration: this.options.streamGeneration, type: 'cancel', operationId: operationIdArg, }); void Promise.resolve().then(() => this.options.operations.send(packet)) .catch((errorArg) => this.fail(errorArg, true)); } private settlePendingOperation( operationIdArg: number, resultArg: interfaces.TControllerBrowserOperationResult, ): void { const pending = this.takePendingOperation(operationIdArg); if (!pending) return; if (resultArg.status === 'failed') { pending.reject(new ControllerBrowserOperationError(resultArg.error.code, resultArg.error.message)); } else if (resultArg.status === 'cancelled') { pending.reject(new DOMException('The browser operation was cancelled.', 'AbortError')); } else { pending.resolve(resultArg); } } private rejectPendingOperation(operationIdArg: number, reasonArg: unknown): boolean { const pending = this.takePendingOperation(operationIdArg); if (!pending) return false; pending.reject(reasonArg); return true; } private takePendingOperation(operationIdArg: number): IPendingBrowserOperation | undefined { const pending = this.pendingOperations.get(operationIdArg); if (!pending) return undefined; this.pendingOperations.delete(operationIdArg); clearTimeout(pending.timeout); if (pending.abortSignal && pending.abortListener) { pending.abortSignal.removeEventListener('abort', pending.abortListener); } return pending; } private rejectAllPending(reasonArg: unknown): void { for (const operationId of [...this.pendingOperations.keys()]) { this.rejectPendingOperation(operationId, reasonArg); } } private fail(errorArg: unknown, recoverableArg: boolean): void { if (this.lifecycle === 'failed' || this.lifecycle === 'stopped') return; this.lifecycle = 'failed'; this.rejectAllPending(errorArg); void this.options.events.abort(errorArg).catch(() => undefined); void this.options.operations.abort(errorArg).catch(() => undefined); if (this.failureReported) return; this.failureReported = true; this.options.onFailure(errorArg, recoverableArg); } private isStopped(): boolean { return this.lifecycle === 'stopped'; } }