/* * SPDX-License-Identifier: AGPL-3.0-or-later * Copyright (C) 2025 Sergej Görzen * This file is part of OmiLAXR. */ using System; using System.Collections.Generic; using System.ComponentModel; using OmiLAXR.TrackingBehaviours.Learner.HeadTracking; using UnityEngine; namespace OmiLAXR.TrackingBehaviours.Learner { /// /// Tracks head movements and detects nodding and shaking gestures using camera position data. /// Uses amplitude analysis and gesture pattern recognition for reliable head gesture detection. /// [AddComponentMenu("OmiLAXR / 3) Tracking Behaviours / Head Tracking Behaviour"), Description("Detects nodding and shaking of the head by using .")] public class HeadTrackingBehaviour : TrackingBehaviour { /// /// Contains comprehensive data about a detected head gesture event. /// public struct HeadTrackingBehaviourArgs { /// /// Duration of the gesture from start to completion. /// public readonly TimeSpan TimeSpan; /// /// Number of gesture cycles detected (e.g., number of nods or shakes). /// public readonly int NumberOfGesture; /// /// Raw HMD position data collected during the gesture. /// public readonly HmdTimedPosition[] HmdPositions; /// /// Modified position data used for shake detection (Y coordinate zeroed). /// public readonly HmdTimedPosition[] HmdPositionsManipulated; /// /// Distance calculations from origin for shake analysis. /// public readonly float[] DistanceHmdPositionsManipulated; /// /// Minimum distance threshold used for gesture detection. /// public readonly double MinDistanceOfGesture; /// /// Initializes head tracking event arguments with gesture data. /// /// Raw position data array /// Modified position data for shake detection /// Distance calculations array /// Minimum detection threshold /// Total gesture duration /// Number of detected gesture cycles public HeadTrackingBehaviourArgs( HmdTimedPosition[] hmdPositions, HmdTimedPosition[] hmdPositionsManipulated, float[] distanceHmdPositionsManipulated, double minDistanceOfGesture, TimeSpan timesSpan, int numberOfGesture) { HmdPositions = hmdPositions; HmdPositionsManipulated = hmdPositionsManipulated; DistanceHmdPositionsManipulated = distanceHmdPositionsManipulated; MinDistanceOfGesture = minDistanceOfGesture; TimeSpan = timesSpan; NumberOfGesture = numberOfGesture; } } /// /// Event triggered when a nodding gesture is detected and completed. /// [Gesture("Head"), Action("Nod")] public TrackingBehaviourEvent OnNodded = new TrackingBehaviourEvent(); /// /// Event triggered when a head shaking gesture is detected and completed. /// [Gesture("Head"), Action("Shake")] public TrackingBehaviourEvent OnShook = new TrackingBehaviourEvent(); /// /// Enable or disable nodding gesture detection. /// public bool trackNoddingEnabled = true; /// /// Enable or disable head shaking gesture detection. /// public bool trackShakingEnabled = true; // Data collection arrays for position tracking private List _hmdPositions = new List(); private List _hmdPositionsManipulated = new List(); private List _distanceHmdPositionsManipulated = new List(); private Vector3 _nullVector; // State tracking booleans for gesture recognition private bool _enoughSamplesCollected = false; private bool _trackingCurrentlyRunning = true; private bool _currentlyNodding = false; private bool _currentlyShaking = false; /*************** * Tunable parameters for gesture detection algorithm * These values control sensitivity and accuracy of gesture recognition * Future optimization may be needed for different use cases ***************/ /// /// Minimum Y-axis movement distance required to detect a nod gesture. /// private const double MinDistanceNodding = 0.01; /// /// Minimum horizontal movement distance required to detect a shake gesture. /// private const double MinDistanceShaking = 0.01; /// /// Minimum movement threshold to distinguish intentional gestures from normal head movement. /// private const double MinNormalHeadMovements = 0.05; /// /// Maximum time window for detecting continuous nodding gestures. /// private readonly TimeSpan _continuousTimeBetweenNods = new TimeSpan(0, 0, 0, 1, 0); /// /// Maximum time window for detecting continuous shaking gestures. /// private readonly TimeSpan _continuousTimeBetweenShakes = new TimeSpan(0, 0, 0, 1, 0); /// /// Minimum data collection time required before gesture detection can begin. /// private readonly TimeSpan _minTimeForGestures = new TimeSpan(0, 0, 0, 1, 0); /// /// Initialize tracking vectors and reference points. /// protected virtual void Start() { _nullVector = Vector3.zero; } /// /// Main update loop for head tracking - processes position data and detects gestures. /// protected virtual void Update() { // Get current HMD position data var addHmdPosition = HmdPosition.Instance.GetHmdPosition(); if (Equals(addHmdPosition, default(HmdTimedPosition))) return; // Add position to raw tracking data _hmdPositions.Add(addHmdPosition); // Create modified position for shake detection (zero Y coordinate for horizontal-only analysis) addHmdPosition.Position.y = 0; _hmdPositionsManipulated.Add(addHmdPosition); // Calculate distance from origin for shake amplitude analysis _distanceHmdPositionsManipulated.Add(Vector3.Distance(_nullVector, _hmdPositionsManipulated[_hmdPositionsManipulated.Count - 1].Position)); // Analyze current data for possible nod or shake gestures NodsOrShakes(ref _currentlyNodding, ref _currentlyShaking, ref _hmdPositions); switch (_currentlyNodding) { // Process detected nodding gesture case true when !_currentlyShaking: Nodding(ref _currentlyNodding, ref _currentlyShaking); break; // Process detected shaking gesture case false when _currentlyShaking: Shaking(ref _currentlyNodding, ref _currentlyShaking); break; } } /// /// Core gesture analysis method that determines if nodding or shaking is occurring. /// /// Reference to current nodding state /// Reference to current shaking state /// Position data to analyze private void NodsOrShakes(ref bool currentlyNodding, ref bool currentlyShaking, ref List hmdPositions) { // Check if we have collected enough samples for reliable analysis HeadUtils.EnoughSamplesCollected(ref hmdPositions, _minTimeForGestures, ref _trackingCurrentlyRunning, ref _enoughSamplesCollected); // Exit early if insufficient data if (!_enoughSamplesCollected) return; // Check if movement is just normal head movement (looking around) if (HeadUtils.NormalHeadMovement("x", hmdPositions, MinNormalHeadMovements) && HeadUtils.NormalHeadMovement("y", hmdPositions, MinNormalHeadMovements) && HeadUtils.NormalHeadMovement("z", hmdPositions, MinNormalHeadMovements)) { // Too much movement on all axes - likely just looking around currentlyNodding = false; currentlyShaking = false; } else { // Analyze for nodding gesture (primarily Y-axis movement) if (HeadUtils.NormalHeadMovement("x", hmdPositions, MinNormalHeadMovements) || HeadUtils.NormalHeadMovement("z", hmdPositions, MinNormalHeadMovements)) { // Too much X or Z movement for a pure nod currentlyNodding = false; } else if (!(HeadUtils.EnoughAmplitude(hmdPositions, MinDistanceNodding, "y"))) { // Insufficient Y-axis amplitude for nodding currentlyNodding = false; } else { currentlyNodding = true; } if (!currentlyNodding) { // Analyze for shaking gesture (horizontal movement) if (HeadUtils.NormalHeadMovement("y", hmdPositions, MinNormalHeadMovements)) { // Too much Y movement for a pure shake currentlyShaking = false; } else if (!HeadUtils.EnoughAmplitude(_distanceHmdPositionsManipulated, MinDistanceShaking)) { // Insufficient horizontal amplitude for shaking currentlyShaking = false; } else { currentlyShaking = true; } } switch (currentlyNodding) { // No gesture detected - clear data for fresh start case false when !currentlyShaking: HeadUtils.ClearAllItems(ref hmdPositions, ref _hmdPositionsManipulated, ref _distanceHmdPositionsManipulated); break; // Conflicting gestures detected - reset (shouldn't happen normally) case true when currentlyShaking: currentlyNodding = false; currentlyShaking = false; HeadUtils.ClearAllItems(ref hmdPositions, ref _hmdPositionsManipulated, ref _distanceHmdPositionsManipulated); break; } } } /// /// Processes gesture completion and clears tracking data for next gesture. /// /// Position data list to clear /// Manipulated position data to clear /// Distance data to clear /// Nodding state to reset /// Shaking state to reset /// Minimum distance used for detection /// Number of gestures detected /// Total gesture duration private static TimeSpan GetLogInfo(ref List hmdPositions, ref List hmdPositionsManipulated, ref List distanceHmdPositionsManipulated, ref bool currentlyNodding, ref bool currentlyShaking, double minDistanceNodding, int numGestures) { var timeSpan = hmdPositions[hmdPositions.Count - 1].Timestamp - hmdPositions[0].Timestamp; currentlyNodding = false; currentlyShaking = false; // Clear all tracking data for next gesture detection HeadUtils.ClearAllItems(ref hmdPositions, ref hmdPositionsManipulated, ref distanceHmdPositionsManipulated); return timeSpan; } /// /// Processes a detected nodding gesture and triggers the OnNodded event when complete. /// /// Current nodding state /// Current shaking state private void Nodding(ref bool currentlyNodding, ref bool currentlyShaking) { if (!trackNoddingEnabled) return; // Check if nodding gesture is still continuing or has finished var positionContinuousNodding = HeadUtils.GetPositionContinuousGesture(_hmdPositions, _continuousTimeBetweenNods); if (positionContinuousNodding == -1) return; // Wait for gesture to complete if (!HeadUtils.GestureFinished(_hmdPositions, positionContinuousNodding, MinDistanceNodding, "y")) return; var numNods = HeadUtils.GetNumOfGestures(_hmdPositions, MinDistanceNodding, "y"); if (numNods <= 0) return; var timeSpan = GetLogInfo(ref _hmdPositions, ref _hmdPositionsManipulated, ref _distanceHmdPositionsManipulated, ref currentlyNodding, ref currentlyShaking, MinDistanceNodding, numNods); OnNodded?.Invoke(this, new HeadTrackingBehaviourArgs(_hmdPositions.ToArray(), _hmdPositionsManipulated.ToArray(), _distanceHmdPositionsManipulated.ToArray(), MinDistanceNodding, timeSpan, numNods)); } /// /// Processes a detected shaking gesture and triggers the OnShook event when complete. /// /// Current nodding state /// Current shaking state private void Shaking(ref bool currentlyNodding, ref bool currentlyShaking) { if (!trackShakingEnabled) return; // Check if shaking gesture is still continuing or has finished var positionContinuousShaking = HeadUtils.GetPositionContinuousGesture(_hmdPositionsManipulated, _continuousTimeBetweenShakes); if (positionContinuousShaking == -1) return; // Wait for gesture to complete if (!HeadUtils.GestureFinished(_distanceHmdPositionsManipulated, positionContinuousShaking, MinDistanceShaking)) return; var numShakes = HeadUtils.GetNumOfGestures(_distanceHmdPositionsManipulated, MinDistanceShaking); if (numShakes <= 0) return; var timeSpan = GetLogInfo(ref _hmdPositions, ref _hmdPositionsManipulated, ref _distanceHmdPositionsManipulated, ref currentlyNodding, ref currentlyShaking, MinDistanceShaking, numShakes); OnShook?.Invoke(this, new HeadTrackingBehaviourArgs(_hmdPositions.ToArray(), _hmdPositionsManipulated.ToArray(), _distanceHmdPositionsManipulated.ToArray(), MinDistanceShaking, timeSpan, numShakes)); } } }