import { RoundedBox, Text } from '@react-three/drei';
import * as THREE from 'three';
import type { VrGestureControls } from './vr';
type TrackedHand = VrGestureControls['state']['trackedHands'][number];
export type VrVirtualHandsFallbackPose = 'demo' | 'none';
export interface VrVirtualHandsProps {
hands: VrGestureControls['state']['trackedHands'];
boardDistance?: number;
scale?: number;
labelMode?: 'full' | 'short' | 'none';
/**
* When `hands` is empty, render a static demo pose so the scene never shows
* an invisible component while MediaPipe is still warming up. Set to
* `'none'` in shipping builds if you only want to render real tracked data.
* Defaults to `'demo'` — safer default for screenshots / smoke tests.
*/
fallbackPose?: VrVirtualHandsFallbackPose;
}
const HAND_CONNECTIONS = [
[0, 1], [1, 2], [2, 3], [3, 4],
[0, 5], [5, 6], [6, 7], [7, 8],
[5, 9], [9, 10], [10, 11], [11, 12],
[9, 13], [13, 14], [14, 15], [15, 16],
[13, 17], [17, 18], [18, 19], [19, 20],
[0, 17],
] as const;
const HAND_JOINTS = [0, 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 17, 18, 19] as const;
const HAND_FINGERTIPS = [4, 8, 12, 16, 20] as const;
// 21-landmark open-palm pose, normalized to MediaPipe's (x,y) in [0,1] with z=0.
// Used as a fallback while real hand tracking is still initializing so that
// VrVirtualHands always has something visible to render.
const DEMO_HAND_LANDMARKS: Array<{ x: number; y: number; z: number }> = [
{ x: 0.50, y: 0.82, z: 0.00 }, // 0 wrist
{ x: 0.40, y: 0.74, z: 0.00 }, // 1 thumb cmc
{ x: 0.33, y: 0.66, z: 0.00 }, // 2 thumb mcp
{ x: 0.29, y: 0.58, z: 0.00 }, // 3 thumb ip
{ x: 0.26, y: 0.51, z: 0.00 }, // 4 thumb tip
{ x: 0.42, y: 0.58, z: 0.00 }, // 5 index mcp
{ x: 0.40, y: 0.48, z: 0.00 }, // 6 index pip
{ x: 0.39, y: 0.40, z: 0.00 }, // 7 index dip
{ x: 0.38, y: 0.33, z: 0.00 }, // 8 index tip
{ x: 0.49, y: 0.56, z: 0.00 }, // 9 middle mcp
{ x: 0.49, y: 0.45, z: 0.00 }, // 10 middle pip
{ x: 0.49, y: 0.36, z: 0.00 }, // 11 middle dip
{ x: 0.49, y: 0.28, z: 0.00 }, // 12 middle tip
{ x: 0.55, y: 0.57, z: 0.00 }, // 13 ring mcp
{ x: 0.57, y: 0.47, z: 0.00 }, // 14 ring pip
{ x: 0.58, y: 0.39, z: 0.00 }, // 15 ring dip
{ x: 0.59, y: 0.31, z: 0.00 }, // 16 ring tip
{ x: 0.62, y: 0.60, z: 0.00 }, // 17 pinky mcp
{ x: 0.64, y: 0.52, z: 0.00 }, // 18 pinky pip
{ x: 0.66, y: 0.44, z: 0.00 }, // 19 pinky dip
{ x: 0.68, y: 0.37, z: 0.00 }, // 20 pinky tip
];
const DEMO_FALLBACK_HANDS: VrGestureControls['state']['trackedHands'] = [
{
handedness: 'right',
confidence: 0.35,
indexTipNorm: DEMO_HAND_LANDMARKS[8],
thumbTipNorm: DEMO_HAND_LANDMARKS[4],
pinchCenterNorm: {
x: (DEMO_HAND_LANDMARKS[4].x + DEMO_HAND_LANDMARKS[8].x) / 2,
y: (DEMO_HAND_LANDMARKS[4].y + DEMO_HAND_LANDMARKS[8].y) / 2,
z: 0,
},
pinchDistance: 0.18,
landmarksNorm: DEMO_HAND_LANDMARKS,
updatedAt: 0,
},
];
function clamp(value: number, min: number, max: number): number {
return Math.min(Math.max(value, min), max);
}
function averagePoints(points: Array<[number, number, number]>): [number, number, number] {
if (points.length === 0) return [0, 0, 0];
const total = points.reduce<[number, number, number]>(
(acc, point) => [acc[0] + point[0], acc[1] + point[1], acc[2] + point[2]],
[0, 0, 0],
);
return [total[0] / points.length, total[1] / points.length, total[2] / points.length];
}
function subtractVec(a: [number, number, number], b: [number, number, number]): [number, number, number] {
return [a[0] - b[0], a[1] - b[1], a[2] - b[2]];
}
function lengthVec(vector: [number, number, number]): number {
return Math.sqrt((vector[0] ** 2) + (vector[1] ** 2) + (vector[2] ** 2));
}
function normalizeVec(vector: [number, number, number]): [number, number, number] {
const length = lengthVec(vector);
if (length <= 1e-5) return [0, 1, 0];
return [vector[0] / length, vector[1] / length, vector[2] / length];
}
function crossVec(a: [number, number, number], b: [number, number, number]): [number, number, number] {
return [
a[1] * b[2] - a[2] * b[1],
a[2] * b[0] - a[0] * b[2],
a[0] * b[1] - a[1] * b[0],
];
}
function midpointVec(a: [number, number, number], b: [number, number, number]): [number, number, number] {
return [(a[0] + b[0]) / 2, (a[1] + b[1]) / 2, (a[2] + b[2]) / 2];
}
function projectTrackedHandPoint(
point: { x: number; y: number; z: number },
boardDistance: number,
handedness: 'left' | 'right' | 'unknown',
scale: number,
): [number, number, number] {
const lateralBias = handedness === 'left' ? -0.42 : handedness === 'right' ? 0.42 : 0;
return [
((point.x - 0.5) * 3.4 + lateralBias) * scale,
(1.7 + (0.5 - point.y) * 2.2) * scale,
(-(boardDistance - 1.1) + clamp(point.z * 1.8, -0.45, 0.45)) * scale,
];
}
function handPalette(handedness: TrackedHand['handedness']) {
if (handedness === 'left') {
return { accent: '#8fdcff', shell: '#17364a', fingertip: '#effbff' };
}
if (handedness === 'right') {
return { accent: '#63ffc7', shell: '#123d2f', fingertip: '#f1fff8' };
}
return { accent: '#ffd78a', shell: '#4d3720', fingertip: '#fff6e8' };
}
function buildPalmQuaternion(
wrist: [number, number, number],
indexBase: [number, number, number],
middleBase: [number, number, number],
pinkyBase: [number, number, number],
) {
const across = normalizeVec(subtractVec(pinkyBase, indexBase));
const forwardSeed = normalizeVec(subtractVec(middleBase, wrist));
const normal = normalizeVec(crossVec(across, forwardSeed));
const forward = normalizeVec(crossVec(normal, across));
const matrix = new THREE.Matrix4();
matrix.makeBasis(
new THREE.Vector3(across[0], across[1], across[2]),
new THREE.Vector3(forward[0], forward[1], forward[2]),
new THREE.Vector3(normal[0], normal[1], normal[2]),
);
return new THREE.Quaternion().setFromRotationMatrix(matrix);
}
function HandBone({
start,
end,
radius,
color,
opacity = 0.96,
}: {
start: [number, number, number];
end: [number, number, number];
radius: number;
color: string;
opacity?: number;
}) {
const startVec = new THREE.Vector3(...start);
const endVec = new THREE.Vector3(...end);
const direction = endVec.clone().sub(startVec);
const length = direction.length();
if (length <= 1e-5) return null;
const midpoint = startVec.clone().add(endVec).multiplyScalar(0.5);
const quaternion = new THREE.Quaternion().setFromUnitVectors(
new THREE.Vector3(0, 1, 0),
direction.normalize(),
);
return (
);
}
function VrVirtualHand({
hand,
boardDistance,
scale,
labelMode,
}: {
hand: TrackedHand;
boardDistance: number;
scale: number;
labelMode: NonNullable;
}) {
const points = hand.landmarksNorm.map((point) =>
projectTrackedHandPoint(point, boardDistance, hand.handedness, scale),
);
if (points.length < 21) return null;
const palette = handPalette(hand.handedness);
const palmCenter = averagePoints([points[0], points[5], points[9], points[13], points[17]]);
const palmWidth = Math.max(0.18, lengthVec(subtractVec(points[17], points[5])) * 0.95);
const palmHeight = Math.max(0.2, lengthVec(subtractVec(points[9], points[0])) * 1.15);
const palmQuaternion = buildPalmQuaternion(points[0], points[5], points[9], points[17]);
const wristAnchor = midpointVec(points[0], averagePoints([points[0], points[5], points[17]]));
const label =
labelMode === 'none'
? ''
: hand.handedness === 'left'
? labelMode === 'short' ? 'L' : 'LEFT HAND'
: hand.handedness === 'right'
? labelMode === 'short' ? 'R' : 'RIGHT HAND'
: 'HAND';
return (
{HAND_CONNECTIONS.map(([from, to], connectionIndex) => {
const isPalmBone = from === 0 || from === 5 || from === 9 || from === 13 || from === 17;
return (
);
})}
{HAND_JOINTS.map((index) => (
))}
{HAND_FINGERTIPS.map((index) => (
))}
{labelMode !== 'none' ? (
{label}
) : null}
);
}
export function VrVirtualHands({
hands,
boardDistance = 4.2,
scale = 1,
labelMode = 'full',
fallbackPose = 'demo',
}: VrVirtualHandsProps) {
const effectiveHands =
hands.length > 0
? hands
: fallbackPose === 'demo'
? DEMO_FALLBACK_HANDS
: [];
return (
<>
{effectiveHands.map((hand, handIndex) => (
))}
>
);
}