/** * @fileoverview GaussianCaptureLoader — ingest a depthprobe RGBD capture into the native 3DGS trainer. * * The HoloScript `com.holoscript.depthprobe` mobile app banks a "sweep": N full-res JPEG frames, each * with an ARCore camera pose (`cameraTransformColumnMajor4x4`) and a low-res depth map. This module * turns that into the trainer's inputs — a depth-fused Gaussian3D INIT (real geometry, not random) * and posed TrainViews — closing the last /critic gap (#1b: capture ingestion) on REAL data. * * THE COORDINATE CONVENTION (the verification-critical part — proven by the round-trip test): * ARCore's pose is camera-TO-world in OpenGL convention (camera looks down −Z, +Y up). forward3D wants * a CV pinhole: world→camera with +Z forward, +Y down (image v increases downward). The bridge is * D = diag(1,−1,−1): * world→CV rotation Vrow = D · Rᵀ (R = pose's upper-left 3×3, camera-to-world) * world→CV translation t = −Vrow · t_cw (t_cw = pose's translation column) * and depth back-projection inverts it: a CV camera point (X,Y,Z) maps to world via the GL point * (X,−Y,−Z): world = R · (X,−Y,−Z) + t_cw. * * Frame decoding (JPEG → pixels) is injected as a callback so the engine takes no image-codec dep — * the caller supplies sharp (Node) or a canvas (browser). */ import { type SplatCamera } from './GaussianTrainer3D'; import { type Gaussian3D } from './GaussianTrainer3D'; import { type TrainView } from './GaussianTrainRunner'; /** ARCore camera-image intrinsics (pixels, for the full image resolution). */ export interface CaptureIntrinsics { fx: number; fy: number; cx: number; cy: number; imageWidth: number; imageHeight: number; } /** One banked frame from the depthprobe manifest. */ export interface CaptureFrame { index: number; /** Column-major 4×4 ARCore camera-to-world transform. */ cameraTransformColumnMajor4x4: number[]; depthWidth: number; depthHeight: number; /** Row-major depth in millimetres, length depthWidth*depthHeight. */ depthMillimeters: number[]; } export interface CaptureManifest { intrinsics: CaptureIntrinsics; frames: CaptureFrame[]; } /** * Convert an ARCore camera-to-world pose into the trainer's SplatCamera (world→CV pinhole), scaled to * a render width (intrinsics scale with resolution; centre-principal stays consistent). */ export declare function cameraFromArcorePose(pose: ArrayLike, intr: CaptureIntrinsics, renderWidth: number): SplatCamera; /** A back-projected world point with its source depth-pixel (for colour lookup). */ export interface DepthPoint { x: number; y: number; z: number; du: number; dv: number; } /** * Back-project a frame's depth map into world-space points (subsampled by `step`, depth gated to * `[minM, maxM]` metres). Uses the full-image intrinsics with depth-pixel→image-pixel scaling. */ export declare function backprojectDepth(frame: CaptureFrame, intr: CaptureIntrinsics, opts?: { step?: number; minM?: number; maxM?: number; }): DepthPoint[]; /** Decode callback: return a frame's RGB as Float64Array (w*h*3, [0,1]). */ export type DecodeFrame = (frameIndex: number, width: number, height: number) => Float64Array; /** * Build a depth-fused Gaussian3D init from all frames: back-project depth, colour each point from the * frame (sampled at depth resolution), isotropic small scale, identity rotation, high opacity. */ export declare function captureToGaussianInit(manifest: CaptureManifest, decode: DecodeFrame, opts?: { step?: number; minM?: number; maxM?: number; scale?: number; opacity?: number; }): Gaussian3D; /** Build posed TrainViews at a render resolution (every `viewStep`-th frame). */ export declare function captureToViews(manifest: CaptureManifest, decode: DecodeFrame, renderWidth: number, renderHeight: number, viewStep?: number): TrainView[]; //# sourceMappingURL=GaussianCaptureLoader.d.ts.map