// SPDX-License-Identifier: Apache-2.0 /** * useIncrementalStitcher — React hook driving the live panorama * engine. * * Lifecycle: * 1. Host calls `useARSession().start()` to put the AR session in * tracking mode. (Works for AR-supported devices only — non-AR * fallback comes in a later phase.) * 2. Host calls `start()` from this hook. The native engine * registers itself as the AR session's frame consumer. * 3. Native emits a state event for every ARFrame the engine * processes (~60 Hz, mostly trivially-skipped). The hook * mirrors this into React state so a `` * or any other consumer can render the live panorama + UX hints. * 4. Host calls `finalize(outputPath)` when the user releases the * shutter; resolves with the final panorama path + stats. * 5. Host calls `cancel()` if the user dismisses the capture. */ import { useCallback, useEffect, useRef, useState } from 'react'; import { getIncrementalNativeModule, incrementalStitcherIsAvailable, subscribeIncrementalState, IncrementalOutcome, type IncrementalState, type IncrementalStartOptions, type IncrementalFinalizeResult, } from './incremental'; export type IncrementalHint = | 'slow-down' | 'scene-uniform' | 'alignment-lost' | 'tracking-poor' | null; export interface UseIncrementalStitcherReturn { /** Whether the native engine is registered. False = no fallback wiring. */ isAvailable: boolean; /** True between successful `start()` and `finalize()`/`cancel()`. */ isRunning: boolean; /** Latest state pushed by the native engine, or null pre-start. */ state: IncrementalState | null; /** * perf-3a change 4 (review fix) — accumulated keyframe thumbnail paths * (raw native paths, capture-ordered, de-duped) for the live strip. * Accumulated per RAW accept event inside the subscription (via a * functional setState updater), NOT off the coalesced `state`, so two * accepts landing in one React batch both survive — the collapsed-render * effect the earlier draft used dropped one. Cleared on start/finalize/ * cancel. Consumers normalise to `file://` at render time. */ keyframeThumbnails: string[]; /** * Convenience: which UX hint to show, derived from the latest * state.outcome. null when nothing should be shown (silent * accepts, skips inside the overlap window). */ hint: IncrementalHint; /** * Convenience: 'high' | 'medium' | null based on the last accept. * Drives confidence-ring rendering in the live preview. */ confidenceLevel: 'high' | 'medium' | null; /** Begin a new capture. Throws if the AR session isn't running. */ start: (options?: IncrementalStartOptions) => Promise; /** * End the capture and write the final panorama. When `outputPath` * is omitted or empty, the native side picks a path under the * app's tmp directory and returns it in the result. * * `captureOrientation` (optional) — pass the user's CURRENT * device orientation at finalize time. The engine prefers this * value over the start-time snapshot for the bake-rotation pass, * so cross-orientation captures (user opened screen in portrait, * captured in landscape) bake correctly. Omit to keep the legacy * behaviour (start-time orientation). */ finalize: ( outputPath?: string, quality?: number, captureOrientation?: string, /** * 2026-05-22 (audit F2b) — measured cumulative translation * magnitude in METRES from the JS-side IMU translation gate. * Used by the auto-resolver in non-AR mode where the engine has * no pose-driven translation source — without this signal the * auto-resolver always picks `panorama` even for shelf scans. * Omit (or pass 0) when no IMU translation data is available * (e.g. in AR mode the native side has its own pose-driven * translation magnitude and prefers that). */ imuTranslationMetres?: number, /** * 2026-06-16 — the EXPLICIT lens the user selected (`'1x'` | `'0.5x'`). * The reliable zoom signal for the high-level warper tree (`'0.5x'` * ultra-wide → spherical). Omit ⇒ treated as `'1x'`. */ lens?: string, ) => Promise; /** Abort the capture without producing output. */ cancel: () => Promise; } /** * Map raw outcome → user-facing hint string. null = no banner. */ function outcomeToHint(outcome: IncrementalOutcome): IncrementalHint { switch (outcome) { case IncrementalOutcome.RejectedTooFar: return 'slow-down'; case IncrementalOutcome.RejectedSceneUniform: return 'scene-uniform'; case IncrementalOutcome.RejectedAlignmentLost: return 'alignment-lost'; case IncrementalOutcome.SkippedTrackingPoor: return 'tracking-poor'; case IncrementalOutcome.AcceptedHigh: case IncrementalOutcome.AcceptedMedium: case IncrementalOutcome.SkippedTooClose: default: return null; } } function outcomeToConfidence( outcome: IncrementalOutcome, ): 'high' | 'medium' | null { if (outcome === IncrementalOutcome.AcceptedHigh) return 'high'; if (outcome === IncrementalOutcome.AcceptedMedium) return 'medium'; return null; } // ── perf-3a change 4 — coalescer pure logic (exported for unit test) ── /** * Sticky-snapshot merge: keep the last-good snapshot fields so the PiP * shows the most recent panorama continuously between accepts (the native * engine emits `panoramaPath` only on accept; reject/skip ticks carry * none — a naive replace would blank the live preview ~60×/s). */ export function stickyMergeIncremental( prev: IncrementalState | null, next: IncrementalState, ): IncrementalState { if (!next.panoramaPath && prev?.panoramaPath) { return { ...next, panoramaPath: prev.panoramaPath, width: prev.width, height: prev.height, }; } return next; } /** * Classify an event as immediate-flush (must render now) vs coalesced. * Immediate: an ACCEPT (a keyframe thumbnail is present, or the accepted * count went up, or the outcome is an `Accepted*`) or a refine-stage * transition (including the terminal `done`/`error` stages). Everything * else — rejects, skips, overlap %, hints — coalesces. */ export function isImmediateIncrementalEvent( next: IncrementalState, prevAcceptedCount: number, prevRefineStage: IncrementalState['refineStage'], ): boolean { const isAccept = !!next.batchKeyframeThumbnailPath || next.acceptedCount > prevAcceptedCount || next.outcome === IncrementalOutcome.AcceptedHigh || next.outcome === IncrementalOutcome.AcceptedMedium; const stage = next.refineStage; const refineTransition = stage != null && stage !== prevRefineStage; return isAccept || refineTransition; } export function useIncrementalStitcher(): UseIncrementalStitcherReturn { const native = getIncrementalNativeModule(); const isAvailable = incrementalStitcherIsAvailable(); const [isRunning, setIsRunning] = useState(false); const [state, setState] = useState(null); // perf-3a change 4 (review fix) — keyframe thumbnails accumulated per RAW // accept event (see the return-type doc). Owned here (not by the consumer) // so a functional updater captures every accept regardless of React // auto-batching of the coalesced `state`. const [keyframeThumbnails, setKeyframeThumbnails] = useState([]); // Keep the latest hint/confidence sticky for a few frames after a // skip — otherwise the UI flickers since SkippedTooClose returns // a "silent" outcome between every accept. We collapse this by // only updating hint when the new outcome is itself a hint or an // accept, leaving non-hint skips alone. const lastHintRef = useRef(null); // ── Coalesced state commits (perf-3a change 4) ─────────────────── // The native engine emits a state event for EVERY processed ARFrame // (~60 Hz in AR mode), most of them SkippedTooClose. Committing each // to React state re-renders the (large) consumer tree per event — on // an old-architecture/RN-0.79 host that is bridge + legacy-UIManager // cost per event. Instead: ACCEPTS and refine-stage transitions flush // IMMEDIATELY (the UI must show a new thumbnail / stage now); rejects, // skips, overlap %, and hints COALESCE into `pendingRef` and commit at // most once per animation frame. The wire contract is untouched (see // `subscribeIncrementalState`); only this default consumer's commit // frequency changes. Bounds renders at ≤ display-rate + one per accept. const pendingRef = useRef(null); const rafRef = useRef(null); const committedRef = useRef(null); const prevAcceptedRef = useRef(0); const prevRefineStageRef = useRef(undefined); const commit = useCallback((next: IncrementalState) => { committedRef.current = next; setState(next); }, []); const cancelPending = useCallback(() => { if (rafRef.current != null) { cancelAnimationFrame(rafRef.current); rafRef.current = null; } pendingRef.current = null; }, []); // Reset the coalescer to its pre-capture state. Called BEFORE // setState(null) on start/finalize/cancel so a late coalesced event // can't resurrect stale state after the capture ended (discard, not // flush — see docs/perf-3a §4.4: nothing act-on-able is ever pending). const resetCoalescer = useCallback(() => { cancelPending(); committedRef.current = null; prevAcceptedRef.current = 0; prevRefineStageRef.current = undefined; setKeyframeThumbnails([]); }, [cancelPending]); useEffect(() => { if (!native) return undefined; const sub = subscribeIncrementalState((nextState) => { // Hint stays sticky, updated per RAW event (ref write, no render); // renders pick it up at the next commit. const newHint = outcomeToHint(nextState.outcome); if (newHint !== null) { lastHintRef.current = newHint; } else if ( nextState.outcome === IncrementalOutcome.AcceptedHigh || nextState.outcome === IncrementalOutcome.AcceptedMedium ) { lastHintRef.current = null; } // Accumulate keyframe thumbnails per RAW event (before any coalescing): // a functional updater is applied in sequence by React, so two accepts // in one batch both append (the collapsed-state effect dropped one). const thumb = nextState.batchKeyframeThumbnailPath; if (thumb) { setKeyframeThumbnails((prev) => (prev.includes(thumb) ? prev : [...prev, thumb])); } // Classify immediate-flush (accept / refine transition) vs coalesced. const immediate = isImmediateIncrementalEvent( nextState, prevAcceptedRef.current, prevRefineStageRef.current, ); prevAcceptedRef.current = nextState.acceptedCount; prevRefineStageRef.current = nextState.refineStage; if (immediate) { // Accept / refine transition supersedes any pending coalesced // event and commits now. cancelPending(); commit(stickyMergeIncremental(committedRef.current, nextState)); return; } // Coalesced (reject / skip / overlap / hint): merge into pending // (per-event sticky merge) and schedule a single rAF commit. const base = pendingRef.current ?? committedRef.current; pendingRef.current = stickyMergeIncremental(base, nextState); if (rafRef.current == null) { rafRef.current = requestAnimationFrame(() => { rafRef.current = null; const p = pendingRef.current; pendingRef.current = null; if (p != null) commit(p); }); } }); return () => { sub?.remove(); cancelPending(); }; }, [native, cancelPending, commit]); const start = useCallback( async (options: IncrementalStartOptions = {}) => { if (!native) { throw new Error( 'useIncrementalStitcher: IncrementalStitcher native ' + 'module is not registered. Ensure the SDK pod has been ' + 'rebuilt against the host app.', ); } // Clear the coalescer + state BEFORE the await (perf-3a §4.4, mirrors // the Camera.tsx 2026-05-23 race fix): the AR GL thread can deliver an // ACCEPT during the start-await window, and a post-await clear would // wipe it (keyframe 0 lost from the strip). Clearing first lets that // accept's immediate commit survive into `state`. resetCoalescer(); setState(null); lastHintRef.current = null; await native.start(options); setIsRunning(true); }, [native, resetCoalescer], ); const finalize = useCallback( async ( outputPath?: string, quality = 90, captureOrientation?: string, imuTranslationMetres?: number, lens?: string, ): Promise => { if (!native) { throw new Error('useIncrementalStitcher: native module unavailable'); } const result = await native.finalize({ outputPath: outputPath ?? '', quality, // 2026-05-18 (iOS cross-orientation fix) — fresh orientation // at finalize time. Engine uses this for bake-rotation // instead of the start-time snapshot. Undefined = keep // legacy start-time behaviour. captureOrientation, // 2026-05-22 (audit F2b) — fold JS-side IMU translation into // the native auto-resolver. In non-AR mode this is the only // translation signal the resolver has (the JS-driver path // doesn't carry tx/ty/tz, so pose-derived translation is 0). // Native side treats it as a magnitude (always ≥ 0). imuTranslationMetres: Math.max(0, imuTranslationMetres ?? 0), // 2026-06-16 — the EXPLICIT lens the user selected ('1x' | '0.5x'). // This is the reliable zoom signal for the high-level warper tree // (0.5x ultra-wide → spherical); deriving zoom from intrinsics FOV was // unreliable (multi-cam 0.5x reaches the ultra-wide by zoom without // changing the reported fx, and the non-AR path may supply fx=0). lens, }); setIsRunning(false); // Clear React state on finalize so the next start doesn't briefly // show stale frame counts / hint banners from the previous capture. // resetCoalescer() first cancels any pending rAF + drops pendingRef // so a late coalesced event can't resurrect stale state (discard, // not flush — perf-3a §4.4). resetCoalescer(); setState(null); lastHintRef.current = null; return result; }, [native, resetCoalescer], ); const cancel = useCallback(async () => { if (!native) return; await native.cancel(); setIsRunning(false); resetCoalescer(); setState(null); lastHintRef.current = null; }, [native, resetCoalescer]); // Cleanup-on-unmount that actually works. The previous version // captured `isRunning` from the initial render (false), so the // cancel never fired. Reading from a ref keeps the latest // value visible at unmount time. const isRunningRef = useRef(false); useEffect(() => { isRunningRef.current = isRunning; }, [isRunning]); useEffect(() => { return () => { if (native && isRunningRef.current) { native.cancel().catch(() => undefined); } }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const confidenceLevel = state ? outcomeToConfidence(state.outcome) : null; return { isAvailable, isRunning, state, keyframeThumbnails, hint: lastHintRef.current, confidenceLevel, start, finalize, cancel, }; }