import type { PagedSplats, SparkRenderer } from "@sparkjsdev/spark"; import { Vector3 } from "three"; import { isDevEnvironment } from "./debug/index.js"; import { NeedleEngineModelLoader, registeredModelLoaderCallbacks } from "./engine_loaders.callbacks.js"; import type { Context } from "./engine_setup.js"; import { DeviceUtilities, getParam } from "./engine_utils.js"; const GAUSSIAN_SPLAT_LOADER_NAME = "Gaussian Splat Loader"; /** `?debugsplats`: once per second, log what Spark's view-change detection sees * (camera world position/scale vs the last-sorted view) — for diagnosing stale splat * sorting from device logs (e.g. scaled AR rigs). The previous flag names keep working. */ const debugSplatSort = getParam("debugsplats") || getParam("debugsplatsort") || getParam("debugspatsort"); // #region Splat quality /** Quality preset for Gaussian Splat rendering/streaming. `"auto"` resolves per device * (mobile → `"low"`, otherwise `"high"`). See {@link resolveGaussianSplatQuality}. */ export type GaussianSplatQuality = "auto" | "high" | "medium" | "low"; /** Concrete Spark settings a {@link GaussianSplatQuality} resolves to. */ export type ResolvedGaussianSplatQuality = { /** Spherical-harmonics bands to fetch/keep (0-3). SH data dominates splat decode * cost, memory and bandwidth — 0 drops view-dependent color but is a step change * in CPU/memory load. */ maxSh: number; /** Minimum milliseconds between splat re-sorts. Spark's own default is 0 * (sort back-to-back whenever the view moved ≥1mm/~2.5°), which keeps a worker * core saturated during any interaction. */ minSortIntervalMs: number; /** Parallel chunk fetch/decode workers while paging (each can saturate a core). */ fetchers: number; /** Spark lodRenderScale — minimum on-screen pixel size for LOD splat selection. * HIGHER is coarser/cheaper: 1 selects splats down to 1px, values up to ~5 are * "often indistinguishable" per Spark's docs while skipping sub-pixel splats. */ lodRenderScale: number; /** Multiplier on Spark's per-device LOD splat budget (`defaultSplatTarget`: * Android 1M, iOS 1.5M, Quest 500k, desktop 2.5M). Lower is cheaper. */ lodSplatScale: number; /** Maximum standard deviations of the Gaussian to rasterize. Rendered quad AREA — and * with it splat fill cost — scales with the square: √8 (Spark default) → √4 halves * splat overdraw. Spark documents √4..√9 as acceptable. */ maxStdDev: number; /** Maximum on-screen pixel radius per splat (Spark default 512). Caps the worst * near-camera overdraw where single splats cover huge screen regions. */ maxPixelRadius: number; /** Spark cone-foveation detail scale at the edge of the foveation cone (default 0.4). * Lower = coarser splats OFF-CENTER, freeing the shared LOD splat budget for the * visible cone — on a phone's ~60-75° AR view the no-foveation cone (90°) covers the * screen, so this costs little visible quality. */ coneFoveate: number; /** Spark cone-foveation detail scale BEHIND the viewer (default 0.2). Lower = coarser. */ behindFoveate: number; /** Framebuffer scale for XR sessions (1 = native XR resolution). Splat content is * soft and tolerates resolution reduction well; fill cost drops with the square. * Applied as a DEFAULT for the next session start (WebXR cannot change it * mid-session) — an explicit {@link NeedleXRSession.framebufferScaleFactor} wins. */ xrFramebufferScale: number; /** Cap on Spark's resident paged-splat pool (0 = Spark's device default: iOS 6.3M, * other mobile 8.4M, desktop 16.8M splats). Bounds streaming MEMORY growth — zooming * into a large paged scene otherwise pages splats in until the tab dies (observed: * iOS Safari page crash on zoom). Only effective before the shared pager is created * (applied at renderer creation; runtime quality changes cannot shrink the pool). */ maxPagedSplats: number; /** Minimum peak alpha for a splat to render at all — Spark culls whole splats below * it in the VERTEX stage (the quad never rasterizes). Spark's default (0.5/255) * culls nothing; raising the floor a few /255 removes the large faint "fog" splats * that cost maximal overdraw for minimal visual contribution. Splat-only fill * lever, used by the deeper adaptation levels. */ minAlpha: number; /** Multiplier on {@link Context.resolutionScaleFactor} — the LAST-RESORT fill * lever, used only by the single deepest adaptation level after every splat-local * knob (LOD, overdraw caps, foveation, alpha floor) is exhausted, and like every * step it must verifiably buy fps or it reverts. Presets keep 1. */ resolutionScale: number; /** XR frame-rate cap (0 = uncapped). AR splat rendering pins the GPU and heats * phones within a minute; running at 30 roughly halves the power draw. Applied * through {@link Context.xrFrameRateLimit} → WebXR updateTargetFrameRate where * the runtime supports it (never by engine-side frame skipping — that flickers * on runtimes without a reprojecting compositor). */ xrMaxFps: number; }; /** Preset table. Sort intervals: Spark's own default is 0 (sort back-to-back every * frame), which profiling showed keeps a worker core saturated during any interaction * (see .github/plans/webgpu.md, "Spark tuning"). Spark measures the interval * START-to-start, so it must exceed the typical sort duration (50-100 ms on large * scenes) to actually idle the worker — 33 ms measurably changed nothing, while 250 ms * showed visibly stale blend order ("flipped" gaussians) during fast orbiting on * desktop. 150 ms is the compromise for high; PlayCanvas discusses ~250 ms as * acceptable, which informs medium/low. Staleness tolerance is helped by Spark's radial * sort default: pure camera rotation doesn't change radial order, only translation does. */ const SPLAT_QUALITY_PRESETS: Record, ResolvedGaussianSplatQuality> = { high: { maxSh: 3, minSortIntervalMs: 150, fetchers: 3, lodRenderScale: 1, lodSplatScale: 1, maxStdDev: Math.sqrt(8), maxPixelRadius: 512, coneFoveate: 0.4, behindFoveate: 0.2, xrFramebufferScale: 1, maxPagedSplats: 0, minAlpha: 0.5 / 255, resolutionScale: 1, xrMaxFps: 0 }, medium: { maxSh: 2, minSortIntervalMs: 250, fetchers: 2, lodRenderScale: 1, lodSplatScale: 1, maxStdDev: Math.sqrt(8), maxPixelRadius: 384, coneFoveate: 0.3, behindFoveate: 0.15, xrFramebufferScale: 0.9, maxPagedSplats: 0, minAlpha: 0.5 / 255, resolutionScale: 1, xrMaxFps: 0 }, // Tuned on a Pixel 9. First iteration (maxSh 0, maxStdDev √4, radius 160) was judged // "significantly worse" with "visibly harsh splat edges" — the overdraw savings // weren't worth the look. Second iteration (√6, radius 256, 400 ms) was judged "much // better", so the render side was raised another notch while keeping the CPU/decode // side (maxSh 1, one fetcher) that made it viable on device. Earlier iterations also // set lodRenderScale to 0.85-0.9 believing lower = fewer splats — the semantics are // the OPPOSITE (it is a minimum pixel size), so that actually selected MORE splats // than default; presets now keep the neutral 1 and leave LOD thinning to the // fps-driven adaptation below (which is what saves AR on mobile). low: { maxSh: 1, minSortIntervalMs: 300, fetchers: 1, lodRenderScale: 1, lodSplatScale: 1, maxStdDev: Math.sqrt(7), maxPixelRadius: 320, coneFoveate: 0.25, behindFoveate: 0.1, xrFramebufferScale: 0.8, maxPagedSplats: 48 * 65536, minAlpha: 0.5 / 255, resolutionScale: 1, xrMaxFps: 30 }, }; /** Resolves a {@link GaussianSplatQuality} to concrete Spark settings. * @param isMobile overrides device detection (defaults to {@link DeviceUtilities.isMobileDevice}) */ export function resolveGaussianSplatQuality(quality: GaussianSplatQuality, isMobile: boolean = DeviceUtilities.isMobileDevice()): ResolvedGaussianSplatQuality { const key = quality === "auto" ? (isMobile ? "low" : "high") : quality; // copy so callers can't mutate the preset table return { ...SPLAT_QUALITY_PRESETS[key] }; } /** Applies a quality preset to the context's SparkRenderer (shared per scene — the last * applied quality wins) and, when a mesh is provided, to that mesh's pager (per-mesh: * SH bands, fetcher count). Safe to call again at runtime, e.g. when the * {@link GaussianSplat} component's `quality` changes. * * `"auto"` additionally enables fps-driven ADAPTATION: the renderer-level LOD settings * degrade in steps while the measured frame rate stays below target (and recover when * it is comfortably above) — see {@link adaptGaussianSplatQuality}. An explicit * `"low"`/`"medium"`/`"high"` is fixed and disables adaptation. */ export function applyGaussianSplatQuality(context: Context, mesh: object | null, quality: GaussianSplatQuality): ResolvedGaussianSplatQuality { const preset = resolveGaussianSplatQuality(quality); const state = splatQualityStates.get(context); if (state) { state.quality = quality; if (quality !== "auto") { // explicit quality is fixed: clear all adaptation state state.level = 0; state.fpsBeforeStep = null; state.downBackoffUntil = 0; state.goodStreak = 0; state.stepFromLevel = null; state.jumpBackoffUntil = 0; state.fpsBeforeProbe = null; state.upBackoffUntil = 0; state.stableStreak = 0; } applyEffectiveSplatQuality(state); } const splatMesh = mesh as { maxSh?: number; paged?: PagedSplats } | null; if (splatMesh) { // SplatMesh.maxSh gates SH evaluation for ALL source types — this is the only // maxSh that reaches non-paged sources (.ply/.spz load every band otherwise) if (typeof splatMesh.maxSh === "number") splatMesh.maxSh = preset.maxSh; const paged = splatMesh.paged; if (paged) { // paged sources additionally gate SH FETCHING — saves bandwidth and memory paged.setMaxSh(preset.maxSh); if (paged.pager) paged.pager.numFetchers = preset.fetchers; } } return preset; } // #region Splat quality adaptation /** Per-level degradation multipliers applied on top of the resolved base preset — * renderer-level LOD/overdraw knobs only. Per-mesh settings (SH bands, fetchers) are * NOT adapted: changing `maxSh` at runtime refetches/frees SH chunks, and oscillating * that would thrash the pager. Level 0 must be neutral (identity). */ const SPLAT_ADAPTATION_LEVELS = [ { lodRenderScale: 1, lodSplatScale: 1, maxPixelRadius: 1, maxStdDevCap: Number.POSITIVE_INFINITY, foveate: 1, minAlphaFloor: 0, resolutionScale: 1 }, { lodRenderScale: 1.5, lodSplatScale: 0.75, maxPixelRadius: 1, maxStdDevCap: Number.POSITIVE_INFINITY, foveate: 0.8, minAlphaFloor: 0, resolutionScale: 1 }, { lodRenderScale: 2.25, lodSplatScale: 0.5, maxPixelRadius: 0.75, maxStdDevCap: Math.sqrt(7), foveate: 0.6, minAlphaFloor: 0, resolutionScale: 1 }, { lodRenderScale: 3.5, lodSplatScale: 0.35, maxPixelRadius: 0.5, maxStdDevCap: Math.sqrt(6), foveate: 0.45, minAlphaFloor: 0, resolutionScale: 1 }, // levels 4-5: LOD/overdraw knobs are exhausted and fps is still bad — the scene is // FILL-bound (device-proven: a zoomed-in splat cloud at level 3 on an iPhone sat at // 15 fps). Raise the splat alpha floor: Spark vertex-culls whole splats below it, // removing the large faint "fog" splats that cost maximal overdraw for minimal // visual contribution. Splat-only — the rest of the scene is untouched. { lodRenderScale: 3.5, lodSplatScale: 0.35, maxPixelRadius: 0.5, maxStdDevCap: Math.sqrt(6), foveate: 0.45, minAlphaFloor: 4 / 255, resolutionScale: 1 }, { lodRenderScale: 3.5, lodSplatScale: 0.35, maxPixelRadius: 0.5, maxStdDevCap: Math.sqrt(6), foveate: 0.45, minAlphaFloor: 10 / 255, resolutionScale: 1 }, // levels 6-7, LAST RESORT: reduce render resolution (0.75 ≈ 44% less fill, // 0.6 ≈ 64% less). Affects the whole canvas, so entering these levels // additionally requires CATASTROPHIC fps (below // {@link ADAPTATION_RESOLUTION_BAD_FRACTION} of target) — a scene running // at 45 fps against target 60 must never go blurry (device log 2026-07-29). { lodRenderScale: 3.5, lodSplatScale: 0.35, maxPixelRadius: 0.5, maxStdDevCap: Math.sqrt(6), foveate: 0.45, minAlphaFloor: 10 / 255, resolutionScale: 0.75 }, { lodRenderScale: 3.5, lodSplatScale: 0.35, maxPixelRadius: 0.5, maxStdDevCap: Math.sqrt(6), foveate: 0.45, minAlphaFloor: 10 / 255, resolutionScale: 0.6 }, ] as const; /** Highest adaptation level (most degraded). */ export const maxGaussianSplatAdaptationLevel = SPLAT_ADAPTATION_LEVELS.length - 1; /** Returns `preset` degraded to the given adaptation `level` (0 = unchanged). Levels * coarsen the LOD selection (`lodRenderScale` — Spark documents values up to ~5 as often * indistinguishable), shrink the LOD splat budget (`lodSplatScale`) and, at higher * levels, tighten the overdraw caps. Pure — does not touch any renderer. */ export function adaptGaussianSplatQuality(preset: ResolvedGaussianSplatQuality, level: number): ResolvedGaussianSplatQuality { const clamped = Math.max(0, Math.min(maxGaussianSplatAdaptationLevel, Math.round(level))); const mul = SPLAT_ADAPTATION_LEVELS[clamped]; return { ...preset, lodRenderScale: preset.lodRenderScale * mul.lodRenderScale, lodSplatScale: preset.lodSplatScale * mul.lodSplatScale, maxPixelRadius: Math.round(preset.maxPixelRadius * mul.maxPixelRadius), maxStdDev: Math.min(preset.maxStdDev, mul.maxStdDevCap), // foveation degrades off-center/behind detail FIRST — where the user isn't looking coneFoveate: Math.max(0.05, preset.coneFoveate * mul.foveate), behindFoveate: Math.max(0.05, preset.behindFoveate * mul.foveate), minAlpha: Math.max(preset.minAlpha, mul.minAlphaFloor), resolutionScale: preset.resolutionScale * mul.resolutionScale, }; } type SplatQualityState = { context: Context; renderer: SparkRenderer; /** last applied quality — adaptation only runs for "auto" */ quality: GaussianSplatQuality; /** current adaptation level (0 = full preset quality) */ level: number; /** time (context.time.time) of the last level change — cooldown reference */ lastChangeTime: number; /** time of the last once-per-second evaluation */ lastEvalTime: number; /** consecutive good evaluations — recovery requires a sustained streak */ goodStreak: number; /** fps captured just before the last degradation step; non-null while that step is * awaiting verification (did it actually improve fps?) */ fpsBeforeStep: number | null; /** until this time, degradation attempts are suppressed (a step proved ineffective) */ downBackoffUntil: number; /** time streaming work (chunk fetch/decode, initial file decode, LOD tree build) was * last observed — verification waits for {@link ADAPTATION_STREAM_SETTLE} of quiet */ lastStreamingTime: number; /** the level a pending degradation step started from — reverts return HERE (matters * for catastrophic jumps, which cross several levels in one verified step) */ stepFromLevel: number | null; /** until this time, catastrophic jumps are suppressed (a jump proved ineffective) */ jumpBackoffUntil: number; /** fps captured just before the last recovery PROBE; non-null while that probe is * awaiting verification (did recovering cost meaningful fps?) */ fpsBeforeProbe: number | null; /** until this time, recovery probes are suppressed (a probe proved too costly) */ upBackoffUntil: number; /** consecutive evaluations in which the controller was at rest (no level change, no * pending verification) — a recovery probe requires a sustained streak */ stableStreak: number; /** the resolutionScale multiplier this controller last applied on top of the app's * own {@link Context.resolutionScaleFactor} (1 = not touching it) */ appliedResolutionScale: number; /** the {@link Context.xrFrameRateLimit} value this controller last applied * (0 = not touching it) */ appliedXrMaxFps: number; /** while `performance.now()` is below this, Spark's per-frame work stays paused — * see the calm-resume visibility handling in {@link ensureSparkRenderer} */ resumeQuietUntil: number; }; /** Splat quality state per context — avoids scene traversal + instanceof for updates. */ const splatQualityStates = new WeakMap(); /** Read-only view of the context's current splat quality state, or `null` if no splat * content has been loaded: the applied quality, the current adaptation level and the * effective (adapted) renderer settings. */ export function getGaussianSplatQualityInfo(context: Context): { quality: GaussianSplatQuality; adaptationLevel: number; effective: ResolvedGaussianSplatQuality } | null { const state = splatQualityStates.get(context); if (!state) return null; return { quality: state.quality, adaptationLevel: state.level, effective: adaptGaussianSplatQuality(resolveGaussianSplatQuality(state.quality), state.level), }; } /** Evaluate once per second; a level change needs this cooldown since the last change. */ const ADAPTATION_EVAL_INTERVAL = 1; const ADAPTATION_CHANGE_COOLDOWN = 2; /** fps below this fraction of target degrades; recovery needs this many consecutive * good (≥95% of target) evaluations. The gap between the two is hysteresis. */ const ADAPTATION_BAD_FRACTION = 0.8; const ADAPTATION_GOOD_FRACTION = 0.95; const ADAPTATION_RECOVERY_STREAK = 4; /** A degradation step is verified after this settle time: it must have raised fps by * {@link ADAPTATION_MIN_IMPROVEMENT}, otherwise the LOD density was not the bottleneck * (compositor-paced AR, fill rate, sort staleness …) and the step is reverted — visual * quality must not be sacrificed for nothing. */ const ADAPTATION_VERIFY_DELAY = 3; const ADAPTATION_MIN_IMPROVEMENT = 1.1; /** After a reverted (ineffective) step, don't try degrading again for this long. */ const ADAPTATION_DOWN_BACKOFF = 30; /** Verification only judges a step after streaming has been quiet for this long. * While chunks are being fetched/decoded (or an initial file decode / LOD tree build * runs), worker decode load depresses fps regardless of render quality — measuring a * degradation in that state conflates decode load with render load and reverts steps * that were genuinely effective (observed in AR: paging is near-continuous while * moving, and effective degradations kept getting reverted + backed off). */ const ADAPTATION_STREAM_SETTLE = 2; /** …but a pending step may only wait for quiet this long. A slow-network AR fill * streams continuously for 30+ s — perpetual postponement froze the controller at * level 1 / 10 fps for an entire session (no-stacking blocks further steps while one * is pending). After the max wait the step is judged despite streaming: the collapse * rule discards fill-corrupted baselines, so misattribution self-corrects. */ const ADAPTATION_VERIFY_MAX_WAIT = 10; /** If fps at verification time fell this far below the recorded baseline, the baseline * is from a different regime (initial fill completed, zoom, thermal throttling) and the * measurement is discarded instead of judged — see the collapse branch below. */ const ADAPTATION_BASELINE_COLLAPSE = 0.8; /** A recovery probe requires the controller to have been at rest for this many * consecutive evaluations (no level changes, no pending verifications). Probing exists * because the good-fps recovery threshold (95% of target) is UNREACHABLE on devices * whose ceiling is below target — an iPhone topping out at ~50 fps against target 60 * stayed at maximum degradation forever, even zoomed out with fps to spare. */ const ADAPTATION_PROBE_STABLE_STREAK = 8; /** A recovery probe is reverted if it cost more than this fraction of the baseline fps * (i.e. quality is only recovered where it is (nearly) free). */ const ADAPTATION_PROBE_MAX_COST = 0.9; /** A degradation step that lowers the render RESOLUTION (level 6) affects the whole * canvas, not just splats — it is only allowed when fps is CATASTROPHIC: below this * fraction of target. Without this gate the controller went blurry at 45 fps against * target 60 (45 < the regular 80% bad threshold — device log 2026-07-29). */ const ADAPTATION_RESOLUTION_BAD_FRACTION = 0.5; /** One adaptation step. Exposed for tests (the fps sample is injectable there — in the * engine loop it is fed from `context.time.smoothedFps`); use * {@link getGaussianSplatQualityInfo} to observe the result. * * Design: degradation must EARN ITS KEEP. Splat scenes are frequently limited by * something LOD density cannot fix — an AR compositor pacing to 30 Hz, fill * rate, the sort worker — and in that situation an fps-only controller ratchets to * maximum degradation while the frame rate stays exactly as bad, ending up ugly AND * sluggish (observed on a Pixel 9 in AR; PlayCanvas at the same fps looks better * simply because it keeps full splat detail). So every degradation step is verified * against the fps it was supposed to buy, reverted if it bought nothing, and further * degradation is backed off — converging on full quality when quality is not the * problem. * @internal */ export function __internalStepGaussianSplatAdaptation(context: Context, sample: { smoothedFps: number; targetFps: number; isInXR: boolean; streaming?: boolean }, now: number): void { const state = splatQualityStates.get(context); if (!state || state.quality !== "auto") return; let level = state.level; if (sample.streaming) state.lastStreamingTime = Math.max(state.lastStreamingTime, now); // verify the last degradation step once it had time to settle: revert if it did // not deliver a measurable fps improvement, and back off further attempts. // Verification waits for streaming to quiet down — decode load would be measured // as "the degradation bought nothing". A step taken DURING streaming thus verifies // against a decode-depressed baseline and may be kept too eagerly; the sustained // good-fps recovery corrects that within a few seconds. const streamQuiet = now - state.lastStreamingTime >= ADAPTATION_STREAM_SETTLE || now - state.lastChangeTime >= ADAPTATION_VERIFY_MAX_WAIT; if (state.fpsBeforeStep !== null && now - state.lastChangeTime >= ADAPTATION_VERIFY_DELAY && streamQuiet) { if (sample.smoothedFps < state.fpsBeforeStep * ADAPTATION_BASELINE_COLLAPSE) { // fps fell far BELOW the baseline: the conditions the baseline was captured // under no longer exist (content finished streaming in, the user zoomed, // thermal throttling) — the comparison says nothing about the step. Discard // it WITHOUT reverting or backing off; the bad-fps path below is free to // degrade further against a fresh baseline. (Reverting here pinned an // iPhone at 11 fps at FULL quality behind the backoff: baseline 40.3 // captured during the initial fill → 17.9 after it completed → judged // "bought no fps", while level 3 had measured 45+ fps minutes earlier.) if (isDevEnvironment()) console.debug(`[SplatQuality] conditions collapsed since the degradation (${state.fpsBeforeStep.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}) — discarding the measurement, keeping level ${level}`); } else if (sample.smoothedFps < state.fpsBeforeStep * ADAPTATION_MIN_IMPROVEMENT) { level = state.stepFromLevel ?? Math.max(0, level - 1); state.downBackoffUntil = now + ADAPTATION_DOWN_BACKOFF; // a failed catastrophic JUMP additionally backs off jumping — otherwise the // deadlocked state (catastrophic + regular backoff) re-jumps immediately if (state.stepFromLevel !== null && state.level - state.stepFromLevel > 1) state.jumpBackoffUntil = now + ADAPTATION_DOWN_BACKOFF; if (isDevEnvironment()) console.debug(`[SplatQuality] degradation bought no fps (${state.fpsBeforeStep.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}) — reverting to level ${level}, backing off ${ADAPTATION_DOWN_BACKOFF}s`); } else if (isDevEnvironment()) console.debug(`[SplatQuality] degradation bought fps (${state.fpsBeforeStep.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}) — keeping level ${level}`); state.fpsBeforeStep = null; state.stepFromLevel = null; } // verify the last recovery probe: quality is only recovered where it is (nearly) // free — if the probe cost measurable fps, restore the degraded level and back off if (state.fpsBeforeProbe !== null && now - state.lastChangeTime >= ADAPTATION_VERIFY_DELAY && streamQuiet) { if (sample.smoothedFps < state.fpsBeforeProbe * ADAPTATION_BASELINE_COLLAPSE) { // conditions collapsed mid-probe (zoom-in, thermal) — restore the degraded // level (deeper is the right direction anyway) but don't punish future // probes: the measurement says nothing about the probe itself level = Math.min(maxGaussianSplatAdaptationLevel, level + 1); if (isDevEnvironment()) console.debug(`[SplatQuality] conditions collapsed during recovery probe (${state.fpsBeforeProbe.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}) — restoring level ${level}`); } else if (sample.smoothedFps < state.fpsBeforeProbe * ADAPTATION_PROBE_MAX_COST) { level = Math.min(maxGaussianSplatAdaptationLevel, level + 1); state.upBackoffUntil = now + ADAPTATION_DOWN_BACKOFF; if (isDevEnvironment()) console.debug(`[SplatQuality] recovery probe cost fps (${state.fpsBeforeProbe.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}) — restoring level ${level}, backing off ${ADAPTATION_DOWN_BACKOFF}s`); } else if (isDevEnvironment()) console.debug(`[SplatQuality] recovery probe kept (fps ${state.fpsBeforeProbe.toFixed(1)} → ${sample.smoothedFps.toFixed(1)}), level ${level}`); state.fpsBeforeProbe = null; } const bad = sample.smoothedFps < sample.targetFps * ADAPTATION_BAD_FRACTION; const good = sample.smoothedFps >= sample.targetFps * ADAPTATION_GOOD_FRACTION; const cooledDown = now - state.lastChangeTime >= ADAPTATION_CHANGE_COOLDOWN; if (bad) { state.goodStreak = 0; // fpsBeforeStep/fpsBeforeProbe === null: never act on top of a step that is // still awaiting verification (postponed verification would otherwise let the // controller ratchet to max degradation with no evidence any step helped) if (cooledDown && now >= state.downBackoffUntil && level === state.level && state.fpsBeforeStep === null && state.fpsBeforeProbe === null && level < maxGaussianSplatAdaptationLevel) { // a step that lowers render RESOLUTION hits the whole canvas — last resort, // only under catastrophic fps (the splat-local levers gate on "bad" alone) const lowersResolution = SPLAT_ADAPTATION_LEVELS[level + 1].resolutionScale < SPLAT_ADAPTATION_LEVELS[level].resolutionScale; if (!lowersResolution || sample.smoothedFps < sample.targetFps * ADAPTATION_RESOLUTION_BAD_FRACTION) { state.fpsBeforeStep = sample.smoothedFps; state.stepFromLevel = level; level++; } } // catastrophic DEADLOCK: fps below half target while regular degradation is // backed off because a rung bought nothing — on a fill-bound view the count // levers all fail verification one by one and the effective deep levers (alpha // floor, resolution) are unreachable rung by rung (device: 10.7 fps at FULL // quality, level 2 judged useless, backoff active). Jump straight to the // deepest level as ONE verified step; an ineffective jump reverts to its // origin level and backs off further jumps. else if (cooledDown && now < state.downBackoffUntil && now >= state.jumpBackoffUntil && sample.smoothedFps < sample.targetFps * ADAPTATION_RESOLUTION_BAD_FRACTION && level === state.level && state.fpsBeforeStep === null && state.fpsBeforeProbe === null && level < maxGaussianSplatAdaptationLevel) { state.fpsBeforeStep = sample.smoothedFps; state.stepFromLevel = level; level = maxGaussianSplatAdaptationLevel; } } else if (good) { state.goodStreak++; if (state.goodStreak >= ADAPTATION_RECOVERY_STREAK && cooledDown && state.fpsBeforeProbe === null && level > 0) { level--; state.goodStreak = 0; } } else state.goodStreak = 0; // recovery PROBE: the good-fps recovery above requires 95% of target, which a device // whose ceiling is below target can never reach — it would stay maximally degraded // forever (measured: iPhone ~50 fps max vs target 60, stuck at level 3 zoomed out). // Once the controller has been at rest for a sustained streak, tentatively recover // one level; the verification above keeps it only if it was (nearly) free. const atRest = level === state.level && state.fpsBeforeStep === null && state.fpsBeforeProbe === null; state.stableStreak = atRest ? state.stableStreak + 1 : 0; // `!bad`: the probe targets devices RESTING below the good-fps ceiling (the middle // band) — recovering quality is never free while fps is outright bad (device log // 2026-07-30: probed 7 → 6 at 10 fps) if (atRest && !bad && level > 0 && state.stableStreak >= ADAPTATION_PROBE_STABLE_STREAK && cooledDown && now >= state.upBackoffUntil && now - state.lastStreamingTime >= ADAPTATION_STREAM_SETTLE) { state.fpsBeforeProbe = sample.smoothedFps; state.stableStreak = 0; level--; } if (level !== state.level) { if (isDevEnvironment()) console.debug(`[SplatQuality] adaptation level ${state.level} → ${level} (fps ${sample.smoothedFps.toFixed(1)}/${sample.targetFps}${level > state.level ? ", verifying fps gain" : state.fpsBeforeProbe !== null ? ", probing recovery" : ""})`); state.level = level; state.lastChangeTime = now; applyEffectiveSplatQuality(state); } } /** Count of in-flight initial splat loads (file decode + LOD tree build) per context — * the non-paged counterpart to the pager's fetch activity for streaming detection. */ const pendingSplatLoadWork = new WeakMap(); /** Whether splat streaming/decode work is currently in flight: paged chunk fetches or * decoded-but-unprocessed chunks on the shared pager, or an initial non-paged load. */ function isSplatStreamingActive(context: Context, state: SplatQualityState): boolean { if ((pendingSplatLoadWork.get(context) ?? 0) > 0) return true; const pager = (state.renderer as unknown as { pager?: { fetchers?: unknown[]; fetched?: unknown[] } }).pager; return !!pager && ((pager.fetchers?.length ?? 0) > 0 || (pager.fetched?.length ?? 0) > 0); } /** How long Spark's per-frame work stays paused after the tab becomes visible again. */ const RESUME_QUIET_MS = 1500; /** Per-frame driver for the fps-based adaptation, registered once per context. */ function updateSplatQualityAdaptation(context: Context, state: SplatQualityState) { // calm resume (see ensureSparkRenderer): release the pause once the quiet time // after becoming visible has passed if (state.resumeQuietUntil !== 0 && performance.now() >= state.resumeQuietUntil) { (state.renderer as unknown as { pauseWork?: boolean }).pauseWork = false; state.resumeQuietUntil = 0; if (isDevEnvironment()) console.debug("[SplatQuality] calm resume over — splat work resumed"); } const now = context.time.time; // streaming is sampled EVERY frame (two array-length reads): a single trickling // fetcher completes and restarts between once-per-second samples, which then read // "not streaming" in the middle of a continuous fill (seen in device logs: f=1 at // :51, f=0 at :52-:53, f=1 at :58 while resident grew the whole time) const streaming = isSplatStreamingActive(context, state); if (streaming) state.lastStreamingTime = Math.max(state.lastStreamingTime, now); if (now - state.lastEvalTime < ADAPTATION_EVAL_INTERVAL) return; state.lastEvalTime = now; if (debugSplatSort) logSplatSortState(context, state); // let the smoothed fps window fill before judging performance if (context.time.frameCount < 60) return; const isInXR = context.isInXR; // XR compositors pace to the session's refresh rate when exposed; 60 is the common // phone AR / desktop baseline. Higher-refresh displays simply read as "good". // An unreachable target is harmless here: ineffective degradation reverts. let targetFps = (isInXR ? context.xrSession?.frameRate : undefined) ?? 60; // a capped XR session can never exceed the cap — chasing the uncapped target // would read as permanently-bad fps and degrade quality in endless futile steps if (isInXR && context.xrFrameRateLimit > 0) targetFps = Math.min(targetFps, context.xrFrameRateLimit); __internalStepGaussianSplatAdaptation(context, { smoothedFps: context.time.smoothedFps, targetFps, isInXR, streaming }, now); } /** `?debugsplats` diagnostics — reads Spark internals (sortedCenter/sortedDir are the * view of the LAST dispatched sort; viewChanged compares the current camera against them). * Logs both the scene camera and, in XR, the renderer's XR camera — Spark sees the latter. */ function logSplatSortState(context: Context, state: SplatQualityState) { const spark = state.renderer as unknown as { sortedCenter?: { x: number; y: number; z: number; distanceTo(v: unknown): number }; activeSplats?: number; lodRenderScale?: number; lodSplatScale?: number; lastLod?: { pos: { distanceTo(v: Vector3): number }; pixelScaleLimit: number; maxSplats: number }; pager?: { pageLru?: { size: number }; pageSplats?: number; maxSplats?: number; fetchers?: unknown[]; fetched?: unknown[] }; }; if (!spark.sortedCenter) return; const report = (label: string, cam: { getWorldPosition(t: Vector3): Vector3; getWorldScale(t: Vector3): Vector3 }) => { const p = cam.getWorldPosition(tempSortDebugPos); const s = cam.getWorldScale(tempSortDebugScale); const scale = (s.x + s.y + s.z) / 3; const dist = spark.sortedCenter ? p.distanceTo(spark.sortedCenter) : -1; const lodDist = spark.lastLod ? spark.lastLod.pos.distanceTo(p) : -1; console.debug(`[SplatSort] ${label} pos=(${p.x.toFixed(4)}, ${p.y.toFixed(4)}, ${p.z.toFixed(4)}) scale=${scale.toFixed(4)} sortedCenterDist=${dist.toFixed(5)} threshold=${(0.001 * scale).toFixed(5)} lodPosDist=${lodDist.toFixed(4)} activeSplats=${spark.activeSplats}`); }; if (context.mainCamera) { report("mainCamera", context.mainCamera); // parent chain — who actually contributes the rig transform? let chain = ""; for (let p = context.mainCamera.parent; p; p = p.parent) { chain += `${p.name || p.type}(s=${p.scale.x.toFixed(2)}) > `; } console.debug(`[SplatSort] mainCamera parents: ${chain || "none"}`); } if (context.isInXR) { const xrCam = context.renderer.xr.getCamera(); if (xrCam) { report("xrCamera", xrCam); // per-eye sub-camera — three may compose these differently than the container if (xrCam.cameras?.[0]) report("xrCamera.eye0", xrCam.cameras[0]); } } // count live splat meshes — duplicated scene content (e.g. loaded template AND an // instantiated copy) doubles fetch, decode and fill and looks like a budget overrun let splatMeshCount = 0; context.scene.traverse(obj => { if (obj.constructor?.name?.includes("SplatMesh")) splatMeshCount++; }); const residentSplats = (spark.pager?.pageLru?.size ?? 0) * (spark.pager?.pageSplats ?? 0); const streaming = isSplatStreamingActive(context, state); console.debug(`[SplatSort] fps=${context.time.smoothedFps.toFixed(1)} splatMeshes=${splatMeshCount} resident=${residentSplats}/${spark.pager?.maxSplats ?? 0} streaming=${streaming}(f=${spark.pager?.fetchers?.length ?? 0},q=${spark.pager?.fetched?.length ?? 0},l=${pendingSplatLoadWork.get(context) ?? 0}) lod renderScale=${spark.lodRenderScale} splatScale=${spark.lodSplatScale} pixelScaleLimit=${spark.lastLod?.pixelScaleLimit?.toExponential(3)} maxSplats=${spark.lastLod?.maxSplats} adaptLevel=${state.level}`); } const tempSortDebugPos = new Vector3(); const tempSortDebugScale = new Vector3(); /** Renderer-level part of a quality preset. `numLodFetchers` seeds the shared SplatPager * the renderer creates on first LOD drive; an already-created pager is updated directly. */ function applyPresetToSparkRenderer(sparkRenderer: SparkRenderer, preset: ResolvedGaussianSplatQuality) { sparkRenderer.minSortIntervalMs = preset.minSortIntervalMs; sparkRenderer.numLodFetchers = preset.fetchers; sparkRenderer.lodRenderScale = preset.lodRenderScale; // Spark's per-device LOD budget (defaultSplatTarget) gives iOS 1.5M splats vs // Android's 1M. On WebKit the larger budget buys heat, not quality: no WebGPU, // App Clip camera/bridge overhead on top, and thermal throttling within a minute // of AR (device-observed) — cap iOS at the Android level. Remove this correction // if the fork's defaultSplatTarget drops iOS to 1M instead. const iosBudgetCorrection = DeviceUtilities.isiOS() ? 2 / 3 : 1; sparkRenderer.lodSplatScale = preset.lodSplatScale * iosBudgetCorrection; sparkRenderer.maxStdDev = preset.maxStdDev; sparkRenderer.maxPixelRadius = preset.maxPixelRadius; sparkRenderer.coneFoveate = preset.coneFoveate; sparkRenderer.behindFoveate = preset.behindFoveate; sparkRenderer.minAlpha = preset.minAlpha; // memory cap for the resident splat pool — only effective before the shared pager // exists (it is created on the first LOD drive with this value) if (preset.maxPagedSplats > 0 && !sparkRenderer.pager) sparkRenderer.maxPagedSplats = preset.maxPagedSplats; if (sparkRenderer.pager) sparkRenderer.pager.numFetchers = preset.fetchers; // XR sessions render splats cheaper at a slightly reduced framebuffer scale (splats // are soft content; fill cost drops quadratically). Registered as the session-start // DEFAULT — an app's explicit NeedleXRSession.framebufferScaleFactor wins. WebXR // cannot change the framebuffer scale mid-session; the adaptation's resolutionScale // reaches running sessions through NeedleXRSession.setDynamicResolutionScale instead. systemXRFramebufferScale = preset.xrFramebufferScale; } let systemXRFramebufferScale: number | undefined; /** Session-start XR framebuffer scale suggested by the splat quality system (undefined * until splat content set a quality). Consumed by NeedleXRSession when no explicit * `NeedleXRSession.framebufferScaleFactor` is set. * @internal */ export function getSystemXRFramebufferScale(): number | undefined { return systemXRFramebufferScale; } let systemXRDynamicResolutionScale: number = 1; /** The dynamic resolution multiplier currently applied by the splat adaptation * (1 = none). Consumed by NeedleXRSession at session start so a session that begins * while the adaptation is at a resolution-reducing level starts reduced right away. * @internal */ export function getSystemXRDynamicResolutionScale(): number { return systemXRDynamicResolutionScale; } /** Applies the state's quality at its current adaptation level to the renderer. */ function applyEffectiveSplatQuality(state: SplatQualityState) { const effective = adaptGaussianSplatQuality(resolveGaussianSplatQuality(state.quality), state.level); applyPresetToSparkRenderer(state.renderer, effective); // the deepest levels trade render resolution for fill (see SPLAT_ADAPTATION_LEVELS) // — applied as a multiplier ON TOP of the app's own resolutionScaleFactor so an // app-configured scale is preserved and restored exactly if (effective.resolutionScale !== state.appliedResolutionScale) { const appBase = state.context.resolutionScaleFactor / state.appliedResolutionScale; state.context.resolutionScaleFactor = appBase * effective.resolutionScale; state.appliedResolutionScale = effective.resolutionScale; systemXRDynamicResolutionScale = effective.resolutionScale; // Context.updateSize is a no-op while an XR session presents (device-verified: // level 6 "bought no fps" in App Clip AR because the canvas never resized) — // running sessions apply the scale through their dynamic path instead // (App Clip canvas resize / WebXR per-view viewport scaling). state.context.xr?.setDynamicResolutionScale(effective.resolutionScale); } // a granted App Clip session sized its canvas before this preset existed — // re-apply so the preset's framebuffer scale reaches the running session state.context.xr?.refreshRenderResolution(); // thermal cap through the context's public knob — only replace the value this // controller set itself so an app-configured xrFrameRateLimit always wins if (effective.xrMaxFps !== state.appliedXrMaxFps) { if (state.context.xrFrameRateLimit === state.appliedXrMaxFps) { state.context.xrFrameRateLimit = effective.xrMaxFps; } state.appliedXrMaxFps = effective.xrMaxFps; } // A changed LOD budget/render scale produces a new splat MAPPING, but Spark only // regenerates the displayed accumulation on view or generator-version changes — with // a still camera the old (larger) selection keeps rendering forever (device-verified: // activeSplats frozen at 932k against a 525k budget, fps pinned). Bump the generator // versions so the new selection actually takes effect. Upstream fix candidate: // needsUpdate should include mapping changes. state.context.scene.traverse(obj => (obj as { updateVersion?: () => void }).updateVersion?.()); } /** Ensures a SparkRenderer exists in the scene. Required for SplatMesh rendering. */ function ensureSparkRenderer(context: Context, ctor: typeof import("@sparkjsdev/spark").SparkRenderer): SparkRenderer { let sparkRenderer: SparkRenderer | undefined; context.scene.traverse(obj => { if (obj instanceof ctor) sparkRenderer = obj as SparkRenderer; }); if (!sparkRenderer) { sparkRenderer = new ctor({ renderer: context.renderer }); context.scene.add(sparkRenderer); } if (!splatQualityStates.has(context)) { const state: SplatQualityState = { context, renderer: sparkRenderer, quality: "auto", level: 0, lastChangeTime: 0, lastEvalTime: 0, goodStreak: 0, fpsBeforeStep: null, downBackoffUntil: 0, lastStreamingTime: -Infinity, stepFromLevel: null, jumpBackoffUntil: 0, fpsBeforeProbe: null, upBackoffUntil: 0, stableStreak: 0, appliedResolutionScale: 1, appliedXrMaxFps: 0, resumeQuietUntil: 0, }; splatQualityStates.set(context, state); // device-appropriate defaults from creation on — a GaussianSplat component with an // explicit quality re-applies over this (see applyGaussianSplatQuality) applyEffectiveSplatQuality(state); // "keep" list — survives context resets, so register exactly once per context context.pre_render_callbacks.push(() => updateSplatQualityAdaptation(context, state)); // CALM RESUME: after the tab was hidden (App Clip round trip, app switch) the // first frames back must not slam the GPU with the full splat pipeline // (accumulator regenerate + LOD re-traverse + sort + paging). iOS Safari tabs // froze PERMANENTLY on resume — main thread wedged in an early post-resume // frame with a healthy GL context (device-logged: 1 rAF tick, one heartbeat, // then silence). Pause Spark's per-frame work while hidden and briefly after // becoming visible; rendering continues from the last generated state. if (typeof document !== "undefined") { document.addEventListener("visibilitychange", () => { const current = splatQualityStates.get(context); if (!current) return; const spark = current.renderer as unknown as { pauseWork?: boolean }; if (document.visibilityState === "hidden") { spark.pauseWork = true; current.resumeQuietUntil = Number.POSITIVE_INFINITY; } else current.resumeQuietUntil = performance.now() + RESUME_QUIET_MS; }); } } else { // scene was cleared/recreated: keep the state but track the current renderer splatQualityStates.get(context)!.renderer = sparkRenderer; } return sparkRenderer; } /** * Whether a URL can serve paged RAD chunks. * * Paging fetches either byte ranges of the root file (`Range: bytes=…`) or sibling * `.radc` URLs derived by rewriting `-lod-0.` → `-lod-N.`. Neither works for `blob:` * or `data:` sources (drag & drop, in-memory files), which have no sibling URLs and * no meaningful range semantics — those fall back to decoding the whole file. * * NOTE: this is a DENY-list on purpose. It used to require an absolute `http(s)://` * URL, which also excluded plain RELATIVE urls — the normal case for project assets * (`resolveUrl` resolves a glb-relative `scene.rad` to `assets/scene.rad` and does not * absolutize). That silently disabled streaming for every scene-authored splat: the * whole file was downloaded and decoded before anything appeared. */ function canPageFrom(url: string): boolean { return !/^(blob:|data:)/i.test(url); } /** * Register the engine's built-in custom model loaders (currently the Gaussian * Splat loader, backed by Spark). * * Called from {@link initNeedleLoader} rather than run as a bare module * side-effect. A side-effect-only `import "./engine_loaders.custom.js"` gets * tree-shaken away when an app bundles the engine — this file is not listed in * the package's `sideEffects` — which silently disabled splat loading (a * dropped `.ply` fell back to the GLTFLoader). A called export cannot be * eliminated. Idempotent, so it is safe to call on every engine init. * @internal */ export function registerBuiltinCustomLoaders() { if (registeredModelLoaderCallbacks.some(e => e.name === GAUSSIAN_SPLAT_LOADER_NAME)) return; NeedleEngineModelLoader.onCreateCustomModelLoader(cb => { switch (cb.mimetype) { case "model/ply": case "model/spz": case "model/splat": case "model/ksplat": case "model/sog": case "model/rad": { const mimetype = cb.mimetype; const context = cb.context; return { name: GAUSSIAN_SPLAT_LOADER_NAME, loadAsync: async (url: string, onProgress?: (event: ProgressEvent) => void) => { // Lazy-load Spark only when a splat is actually loaded, so it stays out of the // main bundle and downloads on demand as its own chunk. const spark = await import("@sparkjsdev/spark"); // Ensure the SparkRenderer is present for splat rendering ensureSparkRenderer(context, spark.SparkRenderer); // RAD ("RADiance field") is Spark's streaming container: it carries a // precomputed LOD tree plus a chunk index, and SparkRenderer pages chunks // in and out by viewpoint against a shared GPU pool. Spark resolves the // `paged` option in the SplatMesh CONSTRUCTOR from `url`, so it cannot be // applied to an already-decoded result — this branch has to build from the // URL instead of going through SplatLoader at all. // // The mesh is returned IMMEDIATELY, before any splat data is resident, and // fills in as chunks arrive. Note there is no terminal "fully loaded" state // for a paged scene: the resident set grows AND shrinks as pages are // evicted (LRU), so callers must not treat first render as completion. // // No `lod: true` here — the LOD tree comes from the file, so requesting one // would trigger the in-browser Web Worker build we are trying to avoid. if (mimetype === "model/rad" && canPageFrom(url)) { return new spark.SplatMesh({ url, paged: true }); } // initial loads count as streaming work for the quality adaptation: // decode + LOD tree build load the workers, which depresses fps for // reasons a render-quality degradation cannot fix pendingSplatLoadWork.set(context, (pendingSplatLoadWork.get(context) ?? 0) + 1); const loadWorkDone = () => pendingSplatLoadWork.set(context, Math.max(0, (pendingSplatLoadWork.get(context) ?? 1) - 1)); const loader = new spark.SplatLoader(); // Pass the file type we already resolved (via the engine's mimetype // detection) to Spark explicitly. Spark's public load/loadAsync // re-derive the type from the URL, which fails for extension-less // `blob:` URLs (drag & drop) — its worker errors with "Unknown file // type" even though the format is known. Mapping mirrors Spark's own // getSplatFileTypeFromPath (`.sog` is the zipped SOG bundle). const fileType = ({ "model/ply": spark.SplatFileType.PLY, "model/spz": spark.SplatFileType.SPZ, "model/splat": spark.SplatFileType.SPLAT, "model/ksplat": spark.SplatFileType.KSPLAT, "model/sog": spark.SplatFileType.PCSOGSZIP, "model/rad": spark.SplatFileType.RAD, } as const)[mimetype]; let result: InstanceType | InstanceType; try { result = await new Promise | InstanceType>((resolve, reject) => { loader.loadInternal({ url, fileType, onLoad: resolve, onError: reject, onProgress: (event: ProgressEvent) => { if (onProgress && event.type === "progress") { const progress = event as { loaded?: number, total?: number }; onProgress(new ProgressEvent("progress", { loaded: progress.loaded ?? 0, total: progress.total ?? 0, lengthComputable: progress.total != null && progress.total > 0, })); } }, }); }); } catch (err) { loadWorkDone(); throw err; } const opts = result instanceof spark.PackedSplats ? { packedSplats: result, lod: true } : { extSplats: result, lod: true }; const mesh = new spark.SplatMesh(opts); // `lod: true` alone does NOT build a LOD tree — Spark never calls // createLodSplats() automatically. Without it a non-RAD source renders its // FULL splat count outside the LOD budget and foveation forever // (device-measured: a .sog contributed a constant +407k splats on top of a // perfectly budget-obeying .rad at every adaptation level). Fire-and-forget: // the mesh renders un-LODed until the tree is ready. mesh.createLodSplats() .catch((err: unknown) => console.warn("[Needle Engine] Building the splat LOD tree failed — rendering without LOD", err)) .finally(loadWorkDone); if (mimetype === "model/ply" || mimetype === "model/sog") { // .ply 3DGS files are trained in OpenCV (Y-down) space; .sog (PlayCanvas // SOG) stores that same raw convention (and PlayCanvas itself flips .sog // on load). Re-orient to three.js/OpenGL (Y-up): 180° about X. // .spz/.splat load upright. mesh.quaternion.set(1, 0, 0, 0); } return mesh; }, parse: () => { throw new Error("Not implemented"); }, } } } return null; }, { name: GAUSSIAN_SPLAT_LOADER_NAME }) }