/** * Camera abstraction layer * Supports Luxonis OAK-D, RealSense, ZED, and simulated cameras * Handles frame capture, depth fusion, and HMAC signing at capture time */ import type { CameraConfig, CameraFrame, CameraType, DepthMap, Pose, StereoConfig, } from '../types/index.js'; import { signFrame, generateNonce } from '../utils/crypto.js'; import { stereoDepth } from '../utils/math.js'; /** Camera driver interface — implement per hardware */ export interface CameraDriver { readonly type: CameraType; initialize(): Promise; captureRGB(): Promise<{ data: Uint8Array; width: number; height: number }>; captureDepth(): Promise; getPose(): Promise; shutdown(): Promise; } /** * Frame capture pipeline with integrity guarantees * * Security properties: * - Every frame is HMAC-SHA256 signed at capture time * - Monotonic sequence numbers detect frame drops/injection * - Timestamps are validated against system clock (±50ms tolerance) */ export class FrameCapture { private sequenceNumber = 0; private lastTimestamp = 0; private readonly maxClockDrift = 50; // ms constructor( private readonly driver: CameraDriver, private readonly config: CameraConfig, private readonly hmacSecret: string, ) { if (!hmacSecret || hmacSecret.length < 32) { throw new Error('HMAC secret must be at least 32 characters'); } } async initialize(): Promise { await this.driver.initialize(); } /** * Capture a signed frame with depth and pose * Returns a CameraFrame with HMAC signature and sequence number */ async capture(): Promise { const timestamp = Date.now(); // Validate timestamp monotonicity (prevents replay of old frames) if (timestamp < this.lastTimestamp) { throw new Error( `Clock went backwards: ${timestamp} < ${this.lastTimestamp}. ` + 'Possible clock manipulation attack.', ); } // Capture RGB + depth in parallel const [rgbResult, depthResult, pose] = await Promise.all([ this.driver.captureRGB(), this.driver.captureDepth(), this.driver.getPose(), ]); // Increment sequence number (monotonic, never resets) this.sequenceNumber++; this.lastTimestamp = timestamp; // Sign the frame at capture time const hmac = signFrame( rgbResult.data, timestamp, this.sequenceNumber, this.hmacSecret, ); const frame: CameraFrame = { id: generateNonce(16), timestamp, rgb: rgbResult.data, width: rgbResult.width, height: rgbResult.height, depth: depthResult.data, pose, hmac, sequenceNumber: this.sequenceNumber, }; return frame; } /** Get current sequence number (for external monitoring) */ getSequenceNumber(): number { return this.sequenceNumber; } async shutdown(): Promise { await this.driver.shutdown(); } } /** * Dual-camera system (foveal + peripheral) * Foveal: OAK-D Pro (20cm-8m, high detail) * Peripheral: OAK-D Long Range (1-35m, wide coverage) */ export class DualCameraSystem { private foveal: FrameCapture | undefined; private peripheral: FrameCapture | undefined; constructor( private readonly fovealDriver: CameraDriver, private readonly peripheralDriver: CameraDriver, private readonly fovealConfig: CameraConfig, private readonly peripheralConfig: CameraConfig, private readonly hmacSecret: string, ) {} async initialize(): Promise { this.foveal = new FrameCapture(this.fovealDriver, this.fovealConfig, this.hmacSecret); this.peripheral = new FrameCapture(this.peripheralDriver, this.peripheralConfig, this.hmacSecret); await Promise.all([ this.foveal.initialize(), this.peripheral.initialize(), ]); } /** Capture from both cameras simultaneously */ async captureStereo(): Promise<{ foveal: CameraFrame; peripheral: CameraFrame }> { if (!this.foveal || !this.peripheral) { throw new Error('Dual camera system not initialized'); } const [foveal, peripheral] = await Promise.all([ this.foveal.capture(), this.peripheral.capture(), ]); return { foveal, peripheral }; } async shutdown(): Promise { await Promise.all([ this.foveal?.shutdown(), this.peripheral?.shutdown(), ]); } } /** * Depth fusion — merge depth maps from multiple cameras * Uses confidence-weighted averaging for overlapping regions */ export function fuseDepthMaps( maps: readonly DepthMap[], ): DepthMap { if (maps.length === 0) throw new Error('No depth maps to fuse'); if (maps.length === 1) return maps[0]!; const reference = maps[0]!; const width = reference.width; const height = reference.height; const fused = new Float32Array(width * height); const totalWeight = new Float32Array(width * height); let minDepth = Infinity; let maxDepth = -Infinity; for (const map of maps) { if (map.width !== width || map.height !== height) { throw new Error('Depth map dimensions must match for fusion'); } for (let i = 0; i < map.data.length; i++) { const depth = map.data[i]!; if (!isFinite(depth) || depth <= 0) continue; const confidence = map.confidence?.[i] ?? 0.5; fused[i]! += depth * confidence; totalWeight[i]! += confidence; if (depth < minDepth) minDepth = depth; if (depth > maxDepth) maxDepth = depth; } } // Normalize by total weight for (let i = 0; i < fused.length; i++) { if (totalWeight[i]! > 0) { fused[i] = fused[i]! / totalWeight[i]!; } } return { data: fused, width, height, minDepth: isFinite(minDepth) ? minDepth : 0, maxDepth: isFinite(maxDepth) ? maxDepth : 0, }; } /** * Compute depth from stereo disparity map * Z = f * B / d (fundamental stereo equation) */ export function disparityToDepth( disparity: Float32Array, config: StereoConfig, width: number, height: number, ): DepthMap { const depth = new Float32Array(disparity.length); const confidence = new Float32Array(disparity.length); for (let i = 0; i < disparity.length; i++) { const d = disparity[i]!; if (d <= 0) { depth[i] = 0; confidence[i] = 0; continue; } const z = stereoDepth(config.intrinsics.fx, config.baseline, d); if (z >= config.minDepth && z <= config.maxDepth) { depth[i] = z; // Confidence decreases with distance (depth noise ∝ z²/fB) confidence[i] = Math.max(0, 1 - (z - config.minDepth) / (config.maxDepth - config.minDepth)); } else { depth[i] = 0; confidence[i] = 0; } } return { data: depth, width, height, minDepth: config.minDepth, maxDepth: config.maxDepth, confidence, }; }