import {WebVisualContainer} from "./web-visual-container.js"; import {zenidLog} from "../../core/zenid-log.js"; import {CameraFacingMode, CameraFrame, ICamera} from "../../interfaces/verification-system-interfaces.js"; import {Inset} from "../../core/generated/zenid-types.generated.js"; import {CameraHelpers} from "../../core/helpers/camera-helpers.js"; import {CameraExplorer} from "./camera-explorer.js"; import {getRelativePosition, getViewportIntersection} from "./helpers.js"; /** * Camera device info with facing mode detection */ export interface CameraDeviceInfo { /** * Device ID */ deviceId: string; /** * Device label */ label: string; /** * Detected facing mode ('user' or 'environment') */ facingMode: CameraFacingMode; /** * Resolution width */ width?: number; /** * Resolution height */ height?: number; /** * Computed score for camera selection */ score?: number; } /** * Web platform implementation of the camera */ export class WebCamera implements ICamera { // Static fields private static readonly DEFAULT_RESOLUTION = 900; // Instance fields facingMode: CameraFacingMode = CameraFacingMode.user; private videoElement!: HTMLVideoElement; private containerElement!: WebVisualContainer; private roiContainer!: WebVisualContainer; private canvas!: HTMLCanvasElement; private mediaStream: MediaStream | null = null; private actualStreamWidth: number = 0; // Actual width of the video stream private actualStreamHeight: number = 0; // Actual height of the video stream private active: boolean = false; private cameraIndex: number = 0; private currentCamera: CameraDeviceInfo | null = null; private hasLoggedCapture: boolean = false; private frameProcessingResolution: number = WebCamera.DEFAULT_RESOLUTION; private cachedFrame: CameraFrame | null = null; private frameRequested: boolean = false; private videoFrameCallbackHandle: number | null = null; private frameProcessorWorker: Worker | null = null; private isProcessing: boolean = false; private orientationChangeHandler: (() => Promise) | null = null; /** * Create a new web camera instance * @param container * @param roiContainer */ constructor(container: WebVisualContainer, roiContainer: WebVisualContainer | null = null) { this.containerElement = container; this.roiContainer = roiContainer || container; this.videoElement = this.containerElement.getElement().querySelector('video') as HTMLVideoElement; // Setup orientation change listener BEFORE early return // This ensures every camera instance has its own listener this.setupOrientationListener(); if (this.videoElement) return; this.videoElement = document.createElement('video'); this.videoElement.setAttribute('playsinline', 'true'); this.videoElement.setAttribute('autoplay', 'true'); this.videoElement.className = 'zenid-camera-video'; this.containerElement.getElement().appendChild(this.videoElement); } // PUBLIC METHODS /** * Switch to the next camera */ async canSwitchCamera(): Promise { return CameraExplorer.canSwitchCamera(this.facingMode); } /** * Clean up resources */ dispose(): void { this.stop(); this.removeOrientationListener(); if (this.videoElement && this.videoElement.parentNode) { this.videoElement.parentNode.removeChild(this.videoElement); } this.videoElement = null as any; } /** * Capture and return the current frame * Pattern: null (request frame) -> frame (request frame) -> frame -> frame... * The actual capture happens in the video frame callback */ async getFrame(): Promise { if (!this.isActive()) { return null; } // If we have cached frame, return it and request next one if (this.cachedFrame) { const frame = this.cachedFrame; this.cachedFrame = null; this.frameRequested = true; this.videoFrameCallbackHandle = this.videoElement.requestVideoFrameCallback(this.onVideoFrame); return frame; } // No cached frame - request one and return null (only if not already in progress) if (!this.frameRequested && !this.isProcessing) { this.frameRequested = true; this.videoFrameCallbackHandle = this.videoElement.requestVideoFrameCallback(this.onVideoFrame); } return null; } /** * Video frame callback - called once when frame is requested * Creates ImageBitmap and sends to worker, does NOT re-schedule itself */ private onVideoFrame = async () => { // Clear handle and flag immediately this.videoFrameCallbackHandle = null; this.frameRequested = false; // Don't send to worker if it's already busy if (this.isProcessing) { return; } // Mark as processing to prevent multiple in-flight requests this.isProcessing = true; if (!this.frameProcessorWorker) { this.isProcessing = false; return; } try { // Get crop and scale parameters const params = this.prepareCropAndScaleParams(this.frameProcessingResolution); if (!params) { this.isProcessing = false; return; } // Create ImageBitmap (hardware-accelerated, minimal main thread time) const bitmap = await createImageBitmap(this.videoElement); // Send to worker for heavy processing (crop, scale, extract pixels) this.frameProcessorWorker.postMessage({ type: 'process', bitmap: bitmap, targetWidth: params.targetWidth, targetHeight: params.targetHeight, sourceRect: { x: params.finalInset.Left, y: params.finalInset.Top, width: params.finalWidth, height: params.finalHeight }, mirror: this.isMirrored() }, [bitmap]); // Transfer bitmap ownership to worker } catch (error) { zenidLog.error('Error in onVideoFrame:', error); this.isProcessing = false; } }; /** * Capture and return the current frame as a JPEG-encoded Blob at sensor resolution * Tries ImageCapture API first (Android), falls back to video element capture */ async getHighResolutionFrameAsEncodedBlob(resolution?: number): Promise { if (!this.isActive() || !this.currentCamera) { return null; } const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent); // Try ImageCapture API first (not on iOS, where it's buggy) // @ts-ignore: ImageCapture might not be available if (!isIOS && typeof ImageCapture !== 'undefined') { try { const track = this.mediaStream!.getVideoTracks()[0]; // @ts-ignore const imageCapture = new ImageCapture(track); const photoCapabilities = await imageCapture.getPhotoCapabilities(); const photoWidth = photoCapabilities.imageWidth?.max || 0; const photoHeight = photoCapabilities.imageHeight?.max || 0; const currentMax = Math.max(this.actualStreamWidth, this.actualStreamHeight); const photoMax = Math.max(photoWidth, photoHeight); zenidLog.info(`ImageCapture available: photo ${photoWidth}×${photoHeight}, stream ${this.actualStreamWidth}×${this.actualStreamHeight}`); // Use ImageCapture only if it provides higher resolution than current stream if (photoMax > currentMax) { // Cap requested resolution to the resolution parameter if provided let requestWidth = photoWidth; let requestHeight = photoHeight; if (resolution) { const aspect = photoWidth / photoHeight; if (photoWidth > photoHeight) { requestWidth = Math.min(photoWidth, resolution); requestHeight = Math.round(requestWidth / aspect); } else { requestHeight = Math.min(photoHeight, resolution); requestWidth = Math.round(requestHeight * aspect); } } zenidLog.info(`Using ImageCapture API for high-res capture: requesting ${requestWidth}×${requestHeight}`); const rawBlob = await imageCapture.takePhoto({ imageWidth: requestWidth, imageHeight: requestHeight }); // Load the captured image so we can crop it const img = new Image(); const imgLoadPromise = new Promise((resolve, reject) => { img.onload = () => resolve(); img.onerror = reject; }); img.src = URL.createObjectURL(rawBlob); await imgLoadPromise; zenidLog.info(`ImageCapture photo loaded: ${img.width}×${img.height}`); // Use helper methods to calculate crop and scale parameters const maxResolution = Math.max(img.width, img.height); const params = this.prepareCropAndScaleParams(maxResolution, img.width, img.height); if (!params) { throw new Error('Invalid crop region for ImageCapture'); } const tempCanvas = document.createElement('canvas'); const tempContext = tempCanvas.getContext('2d', { alpha: false }); if (!tempContext) { throw new Error('Failed to get canvas context for ImageCapture crop'); } tempCanvas.width = params.targetWidth; tempCanvas.height = params.targetHeight; this.drawCroppedAndMirroredFrame(tempContext, tempCanvas, params, img); URL.revokeObjectURL(img.src); const croppedBlob = await new Promise((resolve, reject) => { tempCanvas.toBlob(blob => blob ? resolve(blob) : reject(new Error('toBlob() result was null')), 'image/jpeg', 0.9); }); zenidLog.info(`Captured via ImageCapture (cropped): ${croppedBlob?.size} bytes, ${tempCanvas.width}×${tempCanvas.height}`); return croppedBlob; } else { zenidLog.info('ImageCapture resolution not better than stream, using video element'); } } catch (error) { zenidLog.info('ImageCapture failed, falling back to video element:', error); } } // Fallback: use hidden video element with high-res stream (iOS path) const tempVideo = document.createElement('video'); tempVideo.setAttribute('playsinline', 'true'); tempVideo.setAttribute('autoplay', 'true'); tempVideo.style.position = 'absolute'; tempVideo.style.left = '-9999px'; tempVideo.style.width = '1px'; tempVideo.style.height = '1px'; document.body.appendChild(tempVideo); try { zenidLog.info("Using hidden video element for high-res capture"); const targetResolution = resolution || Math.max(this.currentCamera.width!, this.currentCamera.height!); const highResDimensions = CameraHelpers.getDimensionsInAspectRatio( targetResolution, this.currentCamera.width! / this.currentCamera.height! ); zenidLog.info(`Requesting high-res stream: ${highResDimensions.width}×${highResDimensions.height}`); const highResConstraints: MediaStreamConstraints = { audio: false, video: this.buildVideoConstraints(this.currentCamera.deviceId, highResDimensions.width, highResDimensions.height) }; const highResStream = await WebCamera.getMediaStream(highResConstraints); if (!highResStream) { zenidLog.error('Failed to get high resolution stream'); document.body.removeChild(tempVideo); return null; } tempVideo.srcObject = highResStream; const streamDimensions = await new Promise<{ width: number, height: number }>((resolve, reject) => { const timeout = setTimeout(() => reject(new Error('Timeout waiting for high-res stream')), 5000); tempVideo.onloadedmetadata = async () => { clearTimeout(timeout); try { await tempVideo.play(); zenidLog.info(`High-res stream ready: ${tempVideo.videoWidth}×${tempVideo.videoHeight}`); resolve({ width: tempVideo.videoWidth, height: tempVideo.videoHeight }); } catch (err) { reject(err); } }; }); // Use helper methods to calculate crop and scale parameters const maxResolution = Math.max(streamDimensions.width, streamDimensions.height); const params = this.prepareCropAndScaleParams(maxResolution, streamDimensions.width, streamDimensions.height); if (!params) { throw new Error('Invalid crop region for video element'); } const tempCanvas = document.createElement('canvas'); const tempContext = tempCanvas.getContext('2d', { alpha: false }); if (!tempContext) { throw new Error('Failed to get canvas context for high-res capture'); } tempCanvas.width = params.targetWidth; tempCanvas.height = params.targetHeight; this.drawCroppedAndMirroredFrame(tempContext, tempCanvas, params, tempVideo); zenidLog.info(`Capturing high resolution frame at ${tempCanvas.width}×${tempCanvas.height}`); const blob = await new Promise((resolve, reject) => { tempCanvas.toBlob(blob => blob ? resolve(blob) : reject(new Error('toBlob() result was null')), 'image/jpeg', 0.9); }); if (blob?.type !== 'image/jpeg') { zenidLog.warn(`Failed to encode as image/jpeg, got ${blob?.type} instead`); } highResStream.getTracks().forEach(track => track.stop()); tempVideo.srcObject = null; document.body.removeChild(tempVideo); return blob; } catch (error) { zenidLog.error('Error capturing high resolution frame:', error); if (tempVideo.srcObject) { (tempVideo.srcObject as MediaStream).getTracks().forEach(track => track.stop()); } if (tempVideo.parentNode) { document.body.removeChild(tempVideo); } return null; } } /** * Get the video element (if available) */ getVideoElement(): HTMLVideoElement | null { return this.videoElement; } init() : Promise { return CameraExplorer.init(); } /** * Check if camera is currently active */ isActive(): boolean { return this.active && !!this.mediaStream; } /** * Start the camera */ async start(facingMode?: CameraFacingMode, resolution?: number): Promise { if (!await CameraExplorer.isAnyCameraAvailable()) return false; if (facingMode && this.facingMode !== facingMode) { this.facingMode = facingMode; this.cameraIndex = 0; // Reset to first camera when changing mode this.currentCamera = null; zenidLog.info(`Facing mode set to: ${CameraFacingMode[facingMode]}`); } if (resolution && resolution !== this.frameProcessingResolution) { this.frameProcessingResolution = resolution; zenidLog.info(`Frame processing resolution set to: ${resolution}`); } try { if (!this.currentCamera) { // Get cameras for current facing mode, preferring matching ones but showing all const cameras = await CameraExplorer.getCamerasByFacingMode(this.facingMode); if (cameras.length === 0) { zenidLog.error('No cameras available'); return false; } // Log available cameras for debugging zenidLog.debug(`Available cameras (${cameras.length} total, requested mode: ${this.facingMode}):`); cameras.forEach((cam, idx) => { zenidLog.debug(` ${idx}: ${cam.label} (${cam.facingMode}) - ${cam.width}x${cam.height}, score: ${cam.score || 0}`); }); // Select camera using modulo this.currentCamera = cameras[this.cameraIndex % cameras.length]; zenidLog.debug(`Selected camera: ${this.currentCamera.label} (${this.currentCamera.facingMode})`); } // Use same resolution for both stream and processing const streamResolution = Math.min(this.frameProcessingResolution, Math.max(this.currentCamera.width!, this.currentCamera.height!)); const requestedDimensions = CameraHelpers.getDimensionsInAspectRatio(streamResolution, this.currentCamera.width! / this.currentCamera.height!); zenidLog.info(`Screen: ${window.innerWidth}×${window.innerHeight}, aspect: ${(window.innerWidth / window.innerHeight).toFixed(2)}`); zenidLog.info(`Initial request ${requestedDimensions.width}×${requestedDimensions.height}`); // Build constraints with aspect-matched resolution const constraints: MediaStreamConstraints = { audio: false, video: this.buildVideoConstraints(this.currentCamera.deviceId, requestedDimensions.width, requestedDimensions.height) }; const mediaStream = await WebCamera.getMediaStream(constraints); if (!mediaStream) { zenidLog.error('Failed to get media stream'); return false; } this.mediaStream = mediaStream; // Apply stream to video element this.videoElement.srcObject = this.mediaStream; // Wait for video to be ready await new Promise((resolve) => { if (this.videoElement) { this.videoElement.onloadedmetadata = async () => { if (this.videoElement) { this.videoElement.play().then(() => resolve()); } }; } }); // Store actual stream dimensions (but don't modify captureWidth/captureHeight which are used for constraints) this.actualStreamWidth = this.videoElement.videoWidth; this.actualStreamHeight = this.videoElement.videoHeight; // Log what camera actually provided zenidLog.info(`Camera stream resolution: ${this.actualStreamWidth}×${this.actualStreamHeight}`); zenidLog.info(`Container dimensions: ${this.containerElement.getWidth()}×${this.containerElement.getHeight()}`); // Apply mirroring if using front camera if (this.isMirrored()) { this.videoElement.classList.add('zenid-camera-video-mirrored'); } else { this.videoElement.classList.remove('zenid-camera-video-mirrored'); } this.active = true; // Initialize frame processor worker with message handler const workerUrl = new URL('./web-camera-worker.js', import.meta.url); this.frameProcessorWorker = new Worker(workerUrl, { type: 'module' }); this.frameProcessorWorker.onmessage = (e: MessageEvent) => { if (e.data.type === 'result') { this.cachedFrame = { image: e.data.data, width: e.data.width, height: e.data.height }; this.isProcessing = false; // Clear flag so next frame can be processed } else if (e.data.type === 'error') { zenidLog.error('Worker error:', e.data.error); this.isProcessing = false; // Clear flag on error too } }; return true; } catch (error) { zenidLog.error('Failed to start camera:', error); this.active = false; return false; } } /** * Stop the camera */ async stop(): Promise { // Cancel video frame callback if pending if (this.videoFrameCallbackHandle !== null && this.videoElement) { this.videoElement.cancelVideoFrameCallback(this.videoFrameCallbackHandle); this.videoFrameCallbackHandle = null; } // Terminate worker if (this.frameProcessorWorker) { this.frameProcessorWorker.terminate(); this.frameProcessorWorker = null; } // Clear frame request state this.frameRequested = false; this.isProcessing = false; this.cachedFrame = null; if (this.mediaStream) { // Stop all tracks in the media stream this.mediaStream.getTracks().forEach(track => track.stop()); this.mediaStream = null; } // Clear video source if showing preview if (this.videoElement && this.videoElement.srcObject) { this.videoElement.srcObject = null; } // Reset actual stream dimensions this.actualStreamWidth = 0; this.actualStreamHeight = 0; this.active = false; // NOTE: Don't remove orientation listener here! // The listener should persist across stop/start cycles // It's only removed in dispose() when the instance is destroyed } /** * Switch to the next camera */ async switchCamera(): Promise { if (!await this.canSwitchCamera()) { zenidLog.info(`Cannot switch camera for facing mode "${CameraFacingMode[this.facingMode]}"`); return false; } await this.stop(); this.cameraIndex++; this.currentCamera = null; return await this.start(); } async setTorch(on: boolean): Promise { if (!this.mediaStream) return; const track = this.mediaStream.getVideoTracks()[0]; if (!track) return; try { // @ts-ignore: ImageCapture might not be available in all environments const imageCapture = new ImageCapture(track); await imageCapture.getPhotoCapabilities(); await track.applyConstraints({ advanced: [{ torch: on } as any] } as any); zenidLog.info(`Torch ${on ? 'on' : 'off'}`); } catch (error) { // Silent failure - torch might not be supported } } // PRIVATE METHODS private static async getMediaStream(constraints: MediaStreamConstraints): Promise { try { // Request camera access return await navigator.mediaDevices.getUserMedia(constraints); } catch (error: any) { // Analyze error to provide helpful feedback if (error.name === 'NotAllowedError' || error.name === 'PermissionDeniedError') zenidLog.error('Camera permission denied. Please allow camera access in your browser settings.'); else if (error.name === 'NotFoundError') zenidLog.error('No camera found. Please connect a camera and try again.'); else if (error.name === 'NotReadableError' || error.name === 'TrackStartError') zenidLog.error('Camera is in use by another application or not accessible.'); else if (error.name === 'OverconstrainedError') zenidLog.error('Camera cannot satisfy the requested constraints.'); else if (error.name === 'SecurityError') zenidLog.error('Camera access blocked due to security restrictions. This may be because the page is not loaded over HTTPS.'); else zenidLog.error(`Camera error: ${error.message || 'Unknown error'}`); return undefined; } } private isMirrored(): boolean { return this.facingMode === 'user'; } private buildVideoConstraints(deviceId: string, width: number, height: number): MediaTrackConstraints { return { deviceId: { exact: deviceId }, width: { ideal: width }, height: { ideal: height }, frameRate: { ideal: 30 }, // @ts-ignore: focusDistance is not standard focusDistance: { ideal: 0.3 }, // @ts-ignore: focusMode is not standard focusMode: { ideal: 'continuous' }, zoom: { ideal: 1.0 } }; } private setupOrientationListener(): void { // Remove old listener if exists this.removeOrientationListener(); // Create new handler bound to this instance this.orientationChangeHandler = async () => { if (this.isActive()) { zenidLog.info('Orientation change detected, restarting camera'); await this.stop(); await this.start(); } }; window.addEventListener('orientationchange', this.orientationChangeHandler); } private removeOrientationListener(): void { if (this.orientationChangeHandler) { window.removeEventListener('orientationchange', this.orientationChangeHandler); this.orientationChangeHandler = null; } } /** * Calculate the crop region (insets) based on container and ROI */ private calculateCropRegion(streamWidth?: number, streamHeight?: number): { finalInset: Inset, finalWidth: number, finalHeight: number } | null { const width = streamWidth ?? this.actualStreamWidth; const height = streamHeight ?? this.actualStreamHeight; const containerRect = this.containerElement.getBoundingClientRect(); const roiRect = getViewportIntersection(this.roiContainer.getBoundingClientRect(), containerRect) ?? containerRect; const roiInset = getRelativePosition(roiRect, containerRect); const cameraInset = CameraHelpers.calculateZoomToView(width, height, this.containerElement.getWidth(), this.containerElement.getHeight()); const actualInsetCameraWidth = width - cameraInset.Left! - cameraInset.Right!; const actualInsetCameraHeight = height - cameraInset.Top! - cameraInset.Bottom!; const cameraXRatio = actualInsetCameraWidth / this.containerElement.getWidth(); const cameraYRatio = actualInsetCameraHeight / this.containerElement.getHeight(); const finalInset = { Left: cameraInset.Left! + roiInset.Left! * cameraXRatio, Top: cameraInset.Top! + roiInset.Top! * cameraYRatio, Right: cameraInset.Right! + roiInset.Right! * cameraXRatio, Bottom: cameraInset.Bottom! + roiInset.Bottom! * cameraYRatio }; const finalWidth = width - finalInset.Left! - finalInset.Right!; const finalHeight = height - finalInset.Top! - finalInset.Bottom!; if (finalWidth <= 0 || finalHeight <= 0) { return null; } return { finalInset, finalWidth, finalHeight }; } /** * Prepare crop and scale parameters for frame capture * Calculates both the crop region and the target dimensions after scaling */ private prepareCropAndScaleParams(maxResolution: number, streamWidth?: number, streamHeight?: number): { finalInset: Inset, finalWidth: number, finalHeight: number, targetWidth: number, targetHeight: number } | null { const cropRegion = this.calculateCropRegion(streamWidth, streamHeight); if (!cropRegion) { return null; } const { finalInset, finalWidth, finalHeight } = cropRegion; const scale = Math.min(1, maxResolution / Math.max(finalWidth, finalHeight)); const targetWidth = Math.round(finalWidth * scale); const targetHeight = Math.round(finalHeight * scale); return { finalInset, finalWidth, finalHeight, targetWidth, targetHeight }; } /** * Draw cropped and optionally mirrored frame from video/image to canvas */ private drawCroppedAndMirroredFrame( context: CanvasRenderingContext2D, canvas: HTMLCanvasElement, params: { finalInset: Inset, finalWidth: number, finalHeight: number }, source?: HTMLVideoElement | HTMLImageElement ): void { context.resetTransform(); if (this.isMirrored()) { context.translate(canvas.width, 0); context.scale(-1, 1); } const imageSource = source ?? this.videoElement; context.drawImage(imageSource, params.finalInset.Left!, params.finalInset.Top!, params.finalWidth, params.finalHeight, 0, 0, canvas.width, canvas.height); } }