/**
* React layer: `PoseTrackerProvider` owns a `PoseTrackerClient`, and
* `usePoseTracker` gives screens access to the pipeline plus typed event
* callbacks.
*
* Recommended usage:
* - Mount the provider near the app root — with or without an API token
* (without token, the SDK runs in keypoints-only mode).
* - **Mounting the provider does not load MoveNet** unless `autoPreload` is
* true. On the screen *before* the camera (e.g. home), call `preload()` /
* `warmup()` (default **basic** = model only, no camera permission). For
* WebView, mount a 1×1 `` warmer so
* MoveNet can load — see docs/PRELOAD.md.
* - Call `configure(apiToken)` at any time to upgrade keypoints-only →
* full-engine without restarting the camera pipeline.
*/
import React, {
createContext,
useCallback,
useContext,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { PoseTrackerClient, type PoseTrackerClientOptions, type StartExerciseOptions } from './client';
import type { PoseInputFrame } from './backends/PoseBackend';
import type { ExerciseConfig, SdkManifest } from './types/manifest';
import type { PreloadOptions } from './types/preload';
import type {
ErrorEvent,
PoseTrackerCallbacks,
PoseTrackerEvent,
PoseTrackerEventListener,
PoseTrackerMode,
PoseTrackerStatus,
} from './types/events';
import type { Pose } from './types/pose';
import type { AccelerationDiagnostics, AccelerationState } from './types/acceleration';
import type { QualityState } from './quality/AdaptiveQualityController';
export interface PoseTrackerContextValue {
/**
* The underlying orchestrator. Escape hatch for advanced integrations
* (e.g. `PoseCameraView` reads the active backend and ingests poses
* computed in the vision-camera worklet). Stable across renders.
*/
client: PoseTrackerClient;
status: PoseTrackerStatus;
/** 'keypoints-only' (no validated key/session) or 'full-engine'. */
mode: PoseTrackerMode;
/**
* GPU-acceleration verdict from the warm-up health check: 'unknown' until
* warm-up completes, then 'gpu' | 'cpu-fallback' | 'unavailable'. Can
* downgrade mid-session after an Android GL context loss. See
* docs/ANDROID_GL_ACCELERATION.md.
*/
acceleration: AccelerationState;
/** Full GPU diagnostics (backend, timings, GL renderer, flags, downgrade trail). */
accelerationDiagnostics: AccelerationDiagnostics | null;
/**
* Adaptive camera-quality profile (AdaptiveChoice + crash-guard + FPS
* auto-downgrade). `capturePriority` reflects the host option (`performance`
* default vs `quality`). See docs/ADAPTIVE_QUALITY.md.
*/
quality: QualityState;
/** Last (possibly non-fatal) error; `ready` + keypoints-only can coexist with it. */
error: ErrorEvent | null;
manifest: SdkManifest | null;
/** Empty in keypoints-only mode. */
exercises: ExerciseConfig[];
/**
* Warm-up (handshake + engine + model). Default `coldStart: 'basic'` —
* no getUserMedia. Pass `{ coldStart: 'full' }` to also open the camera.
*/
preload: (options?: PreloadOptions) => Promise;
/** Alias of preload(). */
warmup: (options?: PreloadOptions) => Promise;
/**
* Hot (re)configuration: validates the token and loads the engine without
* touching the camera pipeline. Resolves to true on full-engine mode.
*/
configure: (apiToken?: string) => Promise;
/**
* Start an FSM exercise (manifest) or a custom one (`jump_analysis`,
* `air_time_jump`). Options: `difficulty`, `userHeightCm`, `devicePitchDeg`.
*/
startExercise: (exerciseId: string, options?: StartExerciseOptions) => void;
stopExercise: () => void;
estimatePose: (frame: PoseInputFrame) => Promise;
processFrame: (frame: PoseInputFrame) => Promise;
addEventListener: (listener: PoseTrackerEventListener) => () => void;
/**
* Classic PoseTracker WebView JSON stream (`sendDataToNative` shape).
* Same as `usePoseTracker({ onMessage })`.
*/
addMessageListener: (
listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void,
) => () => void;
}
const PoseTrackerContext = createContext(null);
export interface PoseTrackerProviderProps {
/** Optional: without a token the SDK runs in keypoints-only mode. */
apiToken?: string;
/**
* Remote engine. Default `'v4'`. Pass `'v3'` for the production FSM bundle.
* Unlabeled `startExercise('squat')` still uses the V3 squat FSM unless this
* is explicitly `'v4'`. Merged into `options.engine`; this prop wins.
*/
engine?: 'v3' | 'v4';
options?: PoseTrackerClientOptions;
/** Start preloading as soon as the provider mounts. Default: false. */
autoPreload?: boolean;
children: React.ReactNode;
}
export function PoseTrackerProvider({
apiToken,
engine,
options,
autoPreload = false,
children,
}: PoseTrackerProviderProps): React.JSX.Element {
const clientRef = useRef(null);
if (!clientRef.current) {
clientRef.current = new PoseTrackerClient(apiToken, {
...options,
engine: engine ?? options?.engine,
});
}
const client = clientRef.current;
const [status, setStatus] = useState(client.getStatus());
const [mode, setMode] = useState(client.getMode());
const [error, setError] = useState(null);
const [manifest, setManifest] = useState(null);
const [exercises, setExercises] = useState([]);
const [acceleration, setAcceleration] = useState(client.getAcceleration());
const [accelerationDiagnostics, setAccelerationDiagnostics] =
useState(client.getAccelerationDiagnostics());
const [quality, setQuality] = useState(client.getQualityState());
useEffect(() => {
const offState = client.onStateChange(() => {
setStatus(client.getStatus());
setMode(client.getMode());
setError(client.getError());
setManifest(client.getManifest());
setExercises(client.getAvailableExercises());
setAcceleration(client.getAcceleration());
setAccelerationDiagnostics(client.getAccelerationDiagnostics());
setQuality(client.getQualityState());
});
if (autoPreload) {
client.preload().catch(() => {
// Error is surfaced through status/error state and the error event.
});
}
return () => {
offState();
};
}, [client, autoPreload]);
useEffect(() => {
return () => {
client.dispose().catch(() => {});
};
}, [client]);
const preload = useCallback(
(options?: PreloadOptions) => client.preload(options),
[client],
);
const configureFn = useCallback((token?: string) => client.configure(token), [client]);
const startExercise = useCallback(
(id: string, exerciseOptions?: StartExerciseOptions) => client.startExercise(id, exerciseOptions),
[client],
);
const stopExercise = useCallback(() => client.stopExercise(), [client]);
const estimatePose = useCallback(
(frame: PoseInputFrame) => client.estimatePose(frame),
[client],
);
const processFrame = useCallback(
(frame: PoseInputFrame) => client.processFrame(frame),
[client],
);
const addEventListener = useCallback(
(listener: PoseTrackerEventListener) => client.addEventListener(listener),
[client],
);
const addMessageListener = useCallback(
(listener: (message: import('./events/classicMessage').ClassicNativeMessage) => void) =>
client.addMessageListener(listener),
[client],
);
const value = useMemo(
() => ({
client,
status,
mode,
acceleration,
accelerationDiagnostics,
quality,
error,
manifest,
exercises,
preload,
warmup: preload,
configure: configureFn,
startExercise,
stopExercise,
estimatePose,
processFrame,
addEventListener,
addMessageListener,
}),
[
client,
status,
mode,
acceleration,
accelerationDiagnostics,
quality,
error,
manifest,
exercises,
preload,
configureFn,
startExercise,
stopExercise,
estimatePose,
processFrame,
addEventListener,
addMessageListener,
],
);
return {children};
}
/**
* Access the PoseTracker pipeline and subscribe to typed events.
*
* ```tsx
* const { status, mode, processFrame, configure } = usePoseTracker({
* onKeypoints: (e) => drawSkeleton(e.keypoints), // both modes
* onCounter: (e) => setReps(e.count), // full-engine only
* });
* ```
*/
export function usePoseTracker(callbacks: PoseTrackerCallbacks = {}): PoseTrackerContextValue {
const context = useContext(PoseTrackerContext);
if (!context) {
throw new Error('usePoseTracker must be used within a .');
}
const callbacksRef = useRef(callbacks);
callbacksRef.current = callbacks;
useEffect(() => {
const offEvents = context.addEventListener((event: PoseTrackerEvent) => {
const cb = callbacksRef.current;
switch (event.type) {
case 'initialization':
cb.onInitialization?.(event);
break;
case 'error':
cb.onError?.(event);
break;
case 'warning':
cb.onWarning?.(event);
break;
case 'keypoints':
cb.onKeypoints?.(event);
break;
case 'angles':
cb.onAngles?.(event);
break;
case 'counter':
cb.onCounter?.(event);
break;
case 'posture':
cb.onPosture?.(event);
break;
case 'progression':
cb.onProgression?.(event);
break;
case 'engine_debug':
cb.onEngineDebug?.(event);
break;
case 'recommendations':
cb.onRecommendations?.(event);
break;
case 'form_score':
cb.onFormScore?.(event);
break;
case 'exercise_summary':
cb.onExerciseSummary?.(event);
break;
case 'jump_calibration':
cb.onJumpCalibration?.(event);
break;
case 'jump_started':
cb.onJumpStarted?.(event);
break;
case 'jump_height':
cb.onJumpHeight?.(event);
break;
case 'jump_discarded':
cb.onJumpDiscarded?.(event);
break;
case 'jump_result':
cb.onJumpResult?.(event);
break;
case 'jump_summary':
cb.onJumpSummary?.(event);
break;
case 'quality_changed':
cb.onQualityChanged?.(event);
break;
case 'performance_warning':
cb.onPerformanceWarning?.(event);
break;
case 'runtime_download_progress':
cb.onRuntimeDownloadProgress?.(event);
break;
}
});
const offMessages = context.addMessageListener((message) => {
callbacksRef.current.onMessage?.(message);
});
return () => {
offEvents();
offMessages();
};
}, [context]);
return context;
}