/**
* @license
* SPDX-License-Identifier: Apache-2.0
*/
import { useThree } from '@react-three/fiber';
import type { ThreeElements } from '@react-three/fiber';
import type { ReactNode } from 'react';
import { useEffect, useMemo } from 'react';
import * as THREE from 'three';
import { SplatEnvironmentReadinessStatus } from '../types';
import type {
PairedSplatEnvironmentConfig,
ScenarioMaterialConfig,
SceneConfig,
SplatCollisionProxyConfig,
SplatEnvironmentReadiness,
SplatEnvironmentMetadata,
SplatEnvironmentMetadataInput,
SplatFormat,
SplatRendererKind,
SplatSceneConfigInput,
SplatSceneConfigState,
SplatSceneInput,
ScenarioLightingPreset,
ScenarioLightingProps,
SplatEnvironmentProps,
VisualScenarioConfig,
VisualScenarioExecutionContext,
VisualScenarioExecutionContextInput,
VisualScenarioEffectsProps,
} from '../types';
const DEFAULT_BACKGROUND = '#181a1f';
export function ScenarioLighting({
preset = 'studio',
castShadow = true,
intensity = 1,
}: ScenarioLightingProps) {
if (preset === 'warehouse') {
return (
<>
>
);
}
if (preset === 'low-light') {
return (
<>
>
);
}
if (preset === 'splat') {
return (
<>
>
);
}
return (
<>
>
);
}
export function getScenarioBackground(
preset: ScenarioLightingPreset | undefined,
fallback = DEFAULT_BACKGROUND
) {
if (preset === 'warehouse') return '#20242b';
if (preset === 'low-light') return '#0f1115';
if (preset === 'splat') return '#1b1f24';
return fallback;
}
export function getScenarioCameraPosition(
basePosition: readonly [number, number, number],
scenario?: Pick
): [number, number, number] {
const [x, y, z] = basePosition;
const jitter = scenario?.camera?.jitter ?? 0;
return [
Number((x + jitter * 0.6).toFixed(3)),
Number((y - jitter * 0.4).toFixed(3)),
Number((z + jitter * 0.25).toFixed(3)),
];
}
export function useVisualScenarioExecutionContext({
scenario,
environment,
renderer,
variantId,
enabled,
}: VisualScenarioExecutionContextInput): VisualScenarioExecutionContext {
return useMemo(
() =>
createVisualScenarioExecutionContext({
scenario,
environment,
renderer,
variantId,
enabled,
}),
[enabled, environment, renderer, scenario, variantId]
);
}
export function createVisualScenarioExecutionContext({
scenario,
environment,
renderer,
variantId,
enabled = true,
}: VisualScenarioExecutionContextInput): VisualScenarioExecutionContext {
const pairedEnvironment =
environment ??
(scenario ? createPairedSplatEnvironment(scenario, { renderer }) : undefined);
const splat = scenario?.splat;
const collisionProxy =
pairedEnvironment?.collisionProxy ?? splat?.collisionProxy ?? undefined;
const readiness = getSplatEnvironmentReadiness({
environment: pairedEnvironment,
scenario,
renderer,
enabled,
});
const format =
pairedEnvironment?.splat.format ?? splat?.format ?? readiness.format ?? 'spz';
return {
scenarioId: scenario?.id ?? pairedEnvironment?.id ?? 'visual-scenario',
scenarioLabel:
scenario?.label ?? pairedEnvironment?.label ?? 'Visual scenario',
variantId,
seed: scenario?.seed ?? 0,
lighting: scenario?.lighting ?? 'studio',
environment: scenario?.environment,
camera: {
jitter: scenario?.camera?.jitter ?? 0,
exposure: scenario?.camera?.exposure ?? 1,
noise: scenario?.camera?.noise ?? 0,
blur: scenario?.camera?.blur ?? 0,
},
materials: {
randomizeObjectColors: Boolean(
scenario?.materials?.randomizeObjectColors
),
randomizeTableMaterial: Boolean(
scenario?.materials?.randomizeTableMaterial
),
roughness: scenario?.materials?.roughness,
metalness: scenario?.materials?.metalness,
},
splatEnabled: Boolean(splat?.enabled || pairedEnvironment),
splatSrc: pairedEnvironment?.splat.src ?? splat?.src,
splatFormat: format,
splatRenderer: renderer ?? pairedEnvironment?.splat.renderer,
collisionProxyXmlPath: collisionProxy?.xmlPath,
collisionProxyStatus: collisionProxy?.status,
collisionProxyPrimitives: collisionProxy?.primitives ?? [],
readiness,
transformSource: 'visualScenario.camera',
};
}
export function VisualScenarioEffects(props: VisualScenarioEffectsProps) {
useVisualScenarioEffects(props);
return null;
}
export function useVisualScenarioEffects({
scenario,
enabled = true,
applyBackground = true,
applyFog = true,
applyRenderer = true,
applyMaterials = true,
background,
fogNear,
fogFar,
materialFilter,
}: VisualScenarioEffectsProps) {
const { gl, scene, invalidate } = useThree();
useEffect(() => {
if (!enabled || !scenario) {
return undefined;
}
const previousExposure = gl.toneMappingExposure;
const previousBackground = scene.background;
const previousFog = scene.fog;
const materialSnapshots = new Map<
THREE.Material,
{
color?: THREE.Color;
roughness?: number;
metalness?: number;
}
>();
if (applyRenderer) {
gl.toneMappingExposure = scenario.camera?.exposure ?? 1;
}
if (applyBackground) {
scene.background = new THREE.Color(
background ?? getScenarioBackground(scenario.lighting)
);
}
if (applyFog) {
scene.fog = createScenarioFog(scenario, background, fogNear, fogFar);
}
if (applyMaterials && scenario.materials) {
applyScenarioMaterials(scene, scenario, materialSnapshots, materialFilter);
}
invalidate();
return () => {
gl.toneMappingExposure = previousExposure;
scene.background = previousBackground;
scene.fog = previousFog;
for (const [material, snapshot] of materialSnapshots) {
const mutable = getMutableScenarioMaterial(material);
if (!mutable) continue;
if (snapshot.color) mutable.color.copy(snapshot.color);
if (typeof snapshot.roughness === 'number') {
mutable.roughness = snapshot.roughness;
}
if (typeof snapshot.metalness === 'number') {
mutable.metalness = snapshot.metalness;
}
mutable.needsUpdate = true;
}
invalidate();
};
}, [
applyBackground,
applyFog,
applyMaterials,
applyRenderer,
background,
enabled,
fogFar,
fogNear,
gl,
invalidate,
materialFilter,
scenario,
scene,
]);
}
/**
* Renderer-agnostic Gaussian splat environment boundary.
*
* This component intentionally does not import a specific 3DGS renderer. Pass a
* Spark/GaussianSplats3D object as `children` once the app chooses a renderer,
* and pass MuJoCo/MJCF collision proxy visuals via `collisionProxy`.
*/
export function SplatEnvironment({
environment,
scenario,
renderer,
src,
format,
collisionProxy,
collisionProxyMetadata,
children,
showPlaceholder = true,
...groupProps
}: SplatEnvironmentProps) {
const metadata = useSplatEnvironment({
environment,
scenario,
renderer,
src,
format,
collisionProxy: collisionProxyMetadata,
});
const existingUserData =
typeof groupProps.userData === 'object' && groupProps.userData !== null
? groupProps.userData
: {};
return (
{children}
{children || !showPlaceholder ? null : }
{collisionProxy}
);
}
export function useSplatEnvironment({
environment,
scenario,
renderer,
src,
format,
collisionProxy,
}: SplatEnvironmentMetadataInput): SplatEnvironmentMetadata {
const scenarioEnvironment = useMemo(
() =>
environment ??
(scenario
? createPairedSplatEnvironment(scenario, { renderer })
: undefined),
[environment, renderer, scenario]
);
const resolvedSrc = src ?? scenarioEnvironment?.splat.src ?? scenario?.splat?.src;
const resolvedFormat =
format ??
scenarioEnvironment?.splat.format ??
scenario?.splat?.format ??
'spz';
const resolvedCollisionProxy =
collisionProxy ??
scenarioEnvironment?.collisionProxy ??
scenario?.splat?.collisionProxy ??
undefined;
const readiness = useMemo(
() =>
getSplatEnvironmentReadiness({
environment: scenarioEnvironment,
scenario,
renderer,
src: resolvedSrc,
format: resolvedFormat,
collisionProxy: resolvedCollisionProxy,
}),
[
collisionProxy,
renderer,
resolvedCollisionProxy,
resolvedFormat,
resolvedSrc,
scenario,
scenarioEnvironment,
]
);
return useMemo(
() => ({
src: resolvedSrc,
format: resolvedFormat,
collisionProxy: resolvedCollisionProxy,
readiness,
userData: createSplatEnvironmentUserData({
environment: scenarioEnvironment,
src: resolvedSrc,
format: resolvedFormat,
collisionProxy: resolvedCollisionProxy,
readiness,
}),
}),
[
scenarioEnvironment,
resolvedSrc,
resolvedFormat,
resolvedCollisionProxy,
readiness,
]
);
}
/**
* Resolve a visual scenario's paired splat environment and compose its MJCF
* collision proxy into a MuJoCo scene config.
*
* This hook is renderer-agnostic: apps can use it with Spark, another 3DGS
* renderer, or their own Three scene objects while keeping physics collision
* files paired with the visual splat metadata.
*/
export function useSplatSceneConfig({
sceneConfig,
scenario,
environment,
enabled = true,
renderer,
}: SplatSceneConfigInput): SplatSceneConfigState {
return useMemo(
() =>
createSplatSceneConfig({
sceneConfig,
scenario,
environment,
enabled,
renderer,
}),
[enabled, environment, renderer, scenario, sceneConfig]
);
}
/**
* Resolve a visual scenario's paired splat environment without requiring React.
*
* Use this in codegen, import validators, backend handoff metadata, or app code
* that needs the same behavior as `useSplatSceneConfig` outside a component.
*/
export function createSplatSceneConfig({
sceneConfig,
scenario,
environment,
enabled = true,
renderer,
}: SplatSceneConfigInput): SplatSceneConfigState {
const resolvedEnvironment = enabled
? environment ??
(scenario
? createPairedSplatEnvironment(scenario, { renderer })
: undefined)
: undefined;
const readiness = getSplatEnvironmentReadiness({
environment: resolvedEnvironment,
scenario,
renderer,
enabled,
});
const resolvedSceneConfig = resolvedEnvironment
? withSplatEnvironment(sceneConfig, resolvedEnvironment, { renderer })
: sceneConfig;
return {
environment: resolvedEnvironment,
sceneConfig: resolvedSceneConfig,
enabled:
enabled && readiness.status !== SplatEnvironmentReadinessStatus.Disabled,
readiness,
};
}
export function getSplatEnvironmentReadiness({
environment,
scenario,
renderer,
src,
format,
collisionProxy,
enabled = true,
}: {
environment?: PairedSplatEnvironmentConfig;
scenario?: Pick;
renderer?: SplatRendererKind;
src?: string;
format?: SplatFormat;
collisionProxy?: SplatCollisionProxyConfig;
enabled?: boolean;
}): SplatEnvironmentReadiness {
const splat = scenario?.splat;
const resolvedSrc = src ?? environment?.splat.src ?? splat?.src;
const resolvedFormat =
format ?? environment?.splat.format ?? splat?.format ?? 'spz';
const resolvedRenderer = renderer ?? environment?.splat.renderer;
const resolvedCollisionProxy =
collisionProxy ?? environment?.collisionProxy ?? splat?.collisionProxy ?? undefined;
const requiresCollisionProxy = splat?.requiresCollisionProxy ?? true;
if (!enabled || (splat && splat.enabled === false && !environment)) {
return {
status: SplatEnvironmentReadinessStatus.Disabled,
ready: false,
requiresCollisionProxy,
missing: [],
format: resolvedFormat,
renderer: resolvedRenderer,
message: 'Splat environment is disabled.',
};
}
if (!resolvedSrc) {
return {
status: SplatEnvironmentReadinessStatus.MissingSplat,
ready: false,
requiresCollisionProxy,
missing: ['splat'],
format: resolvedFormat,
renderer: resolvedRenderer,
message: 'Splat environment is missing a visual asset source.',
};
}
if (resolvedRenderer === 'spark' && resolvedFormat !== 'spz') {
return {
status: SplatEnvironmentReadinessStatus.UnsupportedFormat,
ready: false,
requiresCollisionProxy,
missing: [],
format: resolvedFormat,
renderer: resolvedRenderer,
message: `Spark splat rendering requires .spz assets; received ${resolvedFormat}.`,
};
}
if (requiresCollisionProxy && !resolvedCollisionProxy?.xmlPath) {
return {
status: SplatEnvironmentReadinessStatus.MissingCollisionProxy,
ready: false,
requiresCollisionProxy,
missing: ['collisionProxy'],
format: resolvedFormat,
renderer: resolvedRenderer,
message: 'Splat environment is missing paired MJCF collision proxy XML.',
};
}
return {
status: SplatEnvironmentReadinessStatus.Ready,
ready: true,
requiresCollisionProxy,
missing: [],
format: resolvedFormat,
renderer: resolvedRenderer,
message: requiresCollisionProxy
? 'Splat environment has visual asset and collision proxy metadata.'
: 'Splat environment has a visual asset and does not require collision proxy metadata.',
};
}
/**
* Convert a generic visual scenario splat block into a composable splat
* environment config. Visual-only splats are valid; readiness reports whether
* a paired MJCF collision proxy is required before training/physics handoff.
*/
export function createPairedSplatEnvironment(
scenario: Pick,
options: {
id?: string;
label?: string;
description?: string;
renderer?: SplatRendererKind;
} = {}
): PairedSplatEnvironmentConfig | undefined {
const splat = scenario.splat;
const collisionProxy = splat?.collisionProxy;
if (!splat?.enabled || !splat.src) {
return undefined;
}
return {
id: options.id ?? scenario.id ?? 'splat-environment',
label: options.label ?? scenario.label ?? 'Gaussian splat environment',
description:
options.description ??
(scenario.environment
? `Visual ${scenario.environment} splat paired with MJCF collision proxy.`
: undefined),
splat: {
src: splat.src,
format: splat.format ?? 'spz',
renderer: options.renderer,
},
collisionProxy: collisionProxy?.xmlPath
? {
...collisionProxy,
xmlPath: collisionProxy.xmlPath,
}
: undefined,
};
}
function isPairedSplatEnvironment(input: SplatSceneInput): input is PairedSplatEnvironmentConfig {
return (
!!input &&
'splat' in input &&
!!input.splat &&
!('enabled' in input.splat)
);
}
function sceneRelativePath(sceneConfig: SceneConfig, path: string): string {
const src = sceneConfig.src;
if (!src) return path;
const base = src.endsWith('/') ? src : src + '/';
if (path.startsWith(base)) return path.slice(base.length);
return path;
}
function uniquePaths(paths: readonly string[]): string[] {
const seen = new Set();
const result: string[] = [];
for (const path of paths) {
if (seen.has(path)) continue;
seen.add(path);
result.push(path);
}
return result;
}
/**
* Compose a MuJoCo scene config with a paired splat collision proxy.
*
* This keeps the common hybrid setup declarative:
* robot XML remains `sceneFile`, the `.spz` remains a visual-only layer, and
* the paired MJCF collision proxy is added to `environmentFiles`.
*/
export function withSplatEnvironment(
sceneConfig: SceneConfig,
input: SplatSceneInput,
options: { renderer?: SplatRendererKind } = {}
): SceneConfig {
const environment = isPairedSplatEnvironment(input)
? input
: input
? createPairedSplatEnvironment(input, options)
: undefined;
const xmlPath = environment?.collisionProxy?.xmlPath;
if (!xmlPath) return sceneConfig;
return {
...sceneConfig,
environmentFiles: uniquePaths([
...(sceneConfig.environmentFiles ?? []),
sceneRelativePath(sceneConfig, xmlPath),
]),
};
}
export function createSplatEnvironmentUserData({
environment,
src,
format = 'spz',
collisionProxy,
readiness,
}: {
environment?: PairedSplatEnvironmentConfig;
src?: string;
format?: SplatFormat;
collisionProxy?: SplatCollisionProxyConfig;
readiness?: SplatEnvironmentReadiness;
}) {
return {
role: 'splat-environment',
environmentId: environment?.id,
environmentLabel: environment?.label,
splatSrc: src,
splatFormat: format,
splatRenderer: environment?.splat.renderer,
collisionProxyStatus: collisionProxy?.status ?? 'missing',
collisionProxyXmlPath: collisionProxy?.xmlPath,
collisionProxyPrimitives: collisionProxy?.primitives ?? [],
readinessStatus: readiness?.status,
readinessMessage: readiness?.message,
};
}
export function createSparkSplatViewerUrl({
viewerUrl,
splatSrc,
}: {
viewerUrl: string;
splatSrc: string;
}) {
const url = new URL(viewerUrl, 'http://mujoco-react.local');
url.searchParams.set('splat', splatSrc);
return viewerUrl.startsWith('http') ? url.toString() : `${url.pathname}${url.search}`;
}
function SplatPlaceholder() {
return (
);
}
function createScenarioFog(
scenario: VisualScenarioConfig,
background: THREE.ColorRepresentation | undefined,
fogNear: number | undefined,
fogFar: number | undefined
) {
if (scenario.lighting === 'low-light') {
return new THREE.Fog(
background ?? getScenarioBackground(scenario.lighting),
fogNear ?? 2.5,
fogFar ?? 9
);
}
if (scenario.lighting === 'warehouse') {
return new THREE.Fog(
background ?? getScenarioBackground(scenario.lighting),
fogNear ?? 5,
fogFar ?? 16
);
}
return null;
}
function applyScenarioMaterials(
scene: THREE.Scene,
scenario: VisualScenarioConfig,
snapshots: Map<
THREE.Material,
{
color?: THREE.Color;
roughness?: number;
metalness?: number;
}
>,
materialFilter: VisualScenarioEffectsProps['materialFilter']
) {
const materials = scenario.materials;
if (!materials) return;
scene.traverse((object) => {
if (!(object instanceof THREE.Mesh)) {
return;
}
for (const material of normalizeMaterials(object.material)) {
const mutable = getMutableScenarioMaterial(material);
if (!mutable) continue;
if (materialFilter && !materialFilter({ object, material })) continue;
if (!snapshots.has(material)) {
snapshots.set(material, {
color: mutable.color.clone(),
roughness: mutable.roughness,
metalness: mutable.metalness,
});
}
applyScenarioMaterial(mutable, object, scenario, materials);
}
});
}
function applyScenarioMaterial(
material: THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial,
object: THREE.Object3D,
scenario: VisualScenarioConfig,
materials: ScenarioMaterialConfig
) {
const seed = scenario.seed ?? 0;
const objectKey = `${scenario.id ?? 'scenario'}:${object.name}:${material.name}:${seed}`;
const variation = hashToUnitInterval(objectKey);
if (materials.randomizeObjectColors) {
material.color.setHSL(variation, 0.38, 0.42);
}
if (materials.randomizeTableMaterial) {
material.roughness = clamp01(
materials.roughness ?? 0.35 + variation * 0.45
);
material.metalness = clamp01(
materials.metalness ?? variation * 0.12
);
}
material.needsUpdate = true;
}
function normalizeMaterials(
material: THREE.Material | THREE.Material[]
): THREE.Material[] {
return Array.isArray(material) ? material : [material];
}
function getMutableScenarioMaterial(
material: THREE.Material
): THREE.MeshStandardMaterial | THREE.MeshPhysicalMaterial | null {
if (
material instanceof THREE.MeshStandardMaterial ||
material instanceof THREE.MeshPhysicalMaterial
) {
return material;
}
return null;
}
function hashToUnitInterval(value: string) {
let hash = 2166136261;
for (let index = 0; index < value.length; index += 1) {
hash ^= value.charCodeAt(index);
hash = Math.imul(hash, 16777619);
}
return (hash >>> 0) / 4294967295;
}
function clamp01(value: number) {
return Math.max(0, Math.min(1, value));
}
export type SplatCollisionProxy = ReactNode | ThreeElements['group'];