/****************************************************************************** * Copyright (C) Ultraleap, Inc. 2011-2021. * * * * Use subject to the terms of the Apache License 2.0 available at * * http://www.apache.org/licenses/LICENSE-2.0, or another agreement * * between Ultraleap and you, your company or other organization. * ******************************************************************************/ using Leap.Unity.Attributes; using LeapInternal; using System; using System.Collections; using System.Collections.Generic; using UnityEngine; namespace Leap.Unity { using Attributes; /// /// The LeapServiceProvider provides tracked Leap Hand data and images from the device /// via the Leap service running on the client machine. /// public class LeapServiceProvider : LeapProvider { #region Constants /// /// Converts nanoseconds to seconds. /// protected const double NS_TO_S = 1e-6; /// /// Converts seconds to nanoseconds. /// protected const double S_TO_NS = 1e6; /// /// The transform array used for late-latching. /// protected const string HAND_ARRAY_GLOBAL_NAME = "_LeapHandTransforms"; /// /// The maximum number of times the provider will /// attempt to reconnect to the service before giving up. /// #if UNITY_ANDROID protected const int MAX_RECONNECTION_ATTEMPTS = 10; #else protected const int MAX_RECONNECTION_ATTEMPTS = 5; #endif /// /// The number of frames to wait between each /// reconnection attempt. /// #if UNITY_ANDROID protected const int RECONNECTION_INTERVAL = 360; #else protected const int RECONNECTION_INTERVAL = 180; #endif #endregion #region Inspector public enum InteractionVolumeVisualization { None, LeapMotionController, StereoIR170, Automatic } [Tooltip("Displays a representation of the interaction volume in the scene view")] [SerializeField] protected InteractionVolumeVisualization _interactionVolumeVisualization = InteractionVolumeVisualization.Automatic; public InteractionVolumeVisualization SelectedInteractionVolumeVisualization => _interactionVolumeVisualization; public enum FrameOptimizationMode { None, ReuseUpdateForPhysics, ReusePhysicsForUpdate, } [Tooltip("When enabled, the provider will only calculate one leap frame instead of two.")] [SerializeField] protected FrameOptimizationMode _frameOptimization = FrameOptimizationMode.None; public enum PhysicsExtrapolationMode { None, Auto, Manual } [Tooltip("The mode to use when extrapolating physics.\n" + " None - No extrapolation is used at all.\n" + " Auto - Extrapolation is chosen based on the fixed timestep.\n" + " Manual - Extrapolation time is chosen manually by the user.")] [SerializeField] protected PhysicsExtrapolationMode _physicsExtrapolation = PhysicsExtrapolationMode.Auto; [Tooltip("The amount of time (in seconds) to extrapolate the physics data by.")] [SerializeField] protected float _physicsExtrapolationTime = 1.0f / 90.0f; public enum TrackingOptimizationMode { Desktop, Screentop, HMD } [Tooltip("[Service must be >= 4.9.2!] " + "Which tracking mode to request that the service optimize for. " + "(Use the LeapXRServiceProvider for HMD Mode instead of this option!)")] [SerializeField] [EditTimeOnly] protected TrackingOptimizationMode _trackingOptimization = TrackingOptimizationMode.Desktop; [Tooltip("Enable to prevent changes to tracking mode during initialization. The mode the service is currently set to will be retained.")] [SerializeField] private bool _preventInitializingTrackingMode; #if UNITY_2017_3_OR_NEWER [Tooltip("When checked, profiling data from the LeapCSharp worker thread will be used to populate the UnityProfiler.")] [EditTimeOnly] #else [Tooltip("Worker thread profiling requires a Unity version of 2017.3 or greater.")] [Disable] #endif [SerializeField] protected bool _workerThreadProfiling = false; [Tooltip("Which Leap Service API Endpoint to connect to. This is configured on the service with the 'api_namespace' argument.")] [SerializeField] [EditTimeOnly] protected string _serverNameSpace = "Leap Service"; #endregion #region Internal Settings & Memory protected bool _useInterpolation = true; #if SVR protected IntPtr _clockRebaser; protected System.Diagnostics.Stopwatch _stopwatch = new System.Diagnostics.Stopwatch(); #endif // Extrapolate on Android to compensate for the latency introduced by its graphics // pipeline. #if UNITY_ANDROID && !UNITY_EDITOR protected int ExtrapolationAmount = 0; // 15; protected int BounceAmount = 70; #else protected int ExtrapolationAmount = 0; protected int BounceAmount = 0; #endif protected Controller _leapController; protected bool _isDestroyed; protected SmoothedFloat _fixedOffset = new SmoothedFloat(); protected SmoothedFloat _smoothedTrackingLatency = new SmoothedFloat(); protected long _unityToLeapOffset; protected Frame _untransformedUpdateFrame; protected Frame _transformedUpdateFrame; protected Frame _untransformedFixedFrame; protected Frame _transformedFixedFrame; #endregion #region Edit-time Frame Data private Action _onDeviceSafe; /// /// A utility event to get a callback whenever a new device is connected to the service. /// This callback will ALSO trigger a callback upon subscription if a device is already /// connected. /// /// For situations with multiple devices OnDeviceSafe will be dispatched once for each device. /// public event Action OnDeviceSafe { add { if (_leapController != null) { _leapController.Device += (a0, a1) => { value(a1.Device); }; if (_leapController.IsConnected) { foreach (var device in _leapController.Devices) { value(device); } } } _onDeviceSafe += value; } remove { _onDeviceSafe -= value; } } #if UNITY_EDITOR private Frame _backingUntransformedEditTimeFrame = null; private Frame _untransformedEditTimeFrame { get { if (_backingUntransformedEditTimeFrame == null) { _backingUntransformedEditTimeFrame = new Frame(); } return _backingUntransformedEditTimeFrame; } } private Frame _backingEditTimeFrame = null; private Frame _editTimeFrame { get { if (_backingEditTimeFrame == null) { _backingEditTimeFrame = new Frame(); } return _backingEditTimeFrame; } } private Dictionary _cachedLeftHands = new Dictionary(); private Hand _editTimeLeftHand { get { Hand cachedHand = null; if (_cachedLeftHands.TryGetValue(editTimePose, out cachedHand)) { return cachedHand; } else { cachedHand = TestHandFactory.MakeTestHand(isLeft: true, pose: editTimePose); _cachedLeftHands[editTimePose] = cachedHand; return cachedHand; } } } private Dictionary _cachedRightHands = new Dictionary(); private Hand _editTimeRightHand { get { Hand cachedHand = null; if (_cachedRightHands.TryGetValue(editTimePose, out cachedHand)) { return cachedHand; } else { cachedHand = TestHandFactory.MakeTestHand(isLeft: false, pose: editTimePose); _cachedRightHands[editTimePose] = cachedHand; return cachedHand; } } } #endif #endregion #region LeapProvider Implementation public override Frame CurrentFrame { get { #if UNITY_EDITOR if (!Application.isPlaying) { _editTimeFrame.Hands.Clear(); _untransformedEditTimeFrame.Hands.Clear(); _untransformedEditTimeFrame.Hands.Add(_editTimeLeftHand); _untransformedEditTimeFrame.Hands.Add(_editTimeRightHand); transformFrame(_untransformedEditTimeFrame, _editTimeFrame); return _editTimeFrame; } #endif if (_frameOptimization == FrameOptimizationMode.ReusePhysicsForUpdate) { return _transformedFixedFrame; } else { return _transformedUpdateFrame; } } } public override Frame CurrentFixedFrame { get { #if UNITY_EDITOR if (!Application.isPlaying) { _editTimeFrame.Hands.Clear(); _untransformedEditTimeFrame.Hands.Clear(); _untransformedEditTimeFrame.Hands.Add(_editTimeLeftHand); _untransformedEditTimeFrame.Hands.Add(_editTimeRightHand); transformFrame(_untransformedEditTimeFrame, _editTimeFrame); return _editTimeFrame; } #endif if (_frameOptimization == FrameOptimizationMode.ReuseUpdateForPhysics) { return _transformedUpdateFrame; } else { return _transformedFixedFrame; } } } #endregion #region Android Support #if UNITY_ANDROID private AndroidJavaObject _serviceBinder; AndroidJavaClass unityPlayer; AndroidJavaObject activity; AndroidJavaObject context; ServiceCallbacks serviceCallbacks; protected virtual void OnEnable() { CreateAndroidBinding(); } public bool CreateAndroidBinding() { try { if (_serviceBinder != null) { //Check binding status before calling rebind bool bindStatus = _serviceBinder.Call("isBound"); Debug.Log("CreateAndroidBinding - Current service binder status " + bindStatus); if (bindStatus) { return true; } else { _serviceBinder = null; } } if (_serviceBinder == null) { //Get activity and context if (unityPlayer == null) { Debug.Log("CreateAndroidBinding - Getting activity and context"); unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); activity = unityPlayer.GetStatic("currentActivity"); context = activity.Call("getApplicationContext"); serviceCallbacks = new ServiceCallbacks(); } //Create a new service binding Debug.Log("CreateAndroidBinding - Creating a new service binder"); _serviceBinder = new AndroidJavaObject("com.ultraleap.tracking.service_binder.ServiceBinder", context, serviceCallbacks); bool success = _serviceBinder.Call("bind"); if (success) { Debug.Log("CreateAndroidBinding - Binding of service binder complete"); } return true; } } catch (Exception e) { Debug.LogWarning("CreateAndroidBinding - Failed to bind service: " + e.Message); _serviceBinder = null; } return false; } protected virtual void OnDisable() { if (_serviceBinder != null) { Debug.Log("ServiceBinder.unbind..."); _serviceBinder.Call("unbind"); } } #else protected virtual void OnEnable() { } protected virtual void OnDisable() { } #endif #endregion #region Unity Events protected virtual void Reset() { editTimePose = TestHandFactory.TestHandPose.DesktopModeA; } protected virtual void Awake() { _fixedOffset.delay = 0.4f; _smoothedTrackingLatency.SetBlend(0.99f, 0.0111f); } protected virtual void Start() { createController(); _transformedUpdateFrame = new Frame(); _transformedFixedFrame = new Frame(); _untransformedUpdateFrame = new Frame(); _untransformedFixedFrame = new Frame(); } protected virtual void Update() { if (_workerThreadProfiling) { LeapProfiling.Update(); } if (!checkConnectionIntegrity()) { return; } #if UNITY_EDITOR if (UnityEditor.EditorApplication.isCompiling) { UnityEditor.EditorApplication.isPlaying = false; Debug.LogWarning("Unity hot reloading not currently supported. Stopping Editor Playback."); return; } #endif _fixedOffset.Update(Time.time - Time.fixedTime, Time.deltaTime); if (_frameOptimization == FrameOptimizationMode.ReusePhysicsForUpdate) { DispatchUpdateFrameEvent(_transformedFixedFrame); return; } #if SVR if (_clockRebaser != IntPtr.Zero) { eLeapRS result = LeapC.UpdateRebase(_clockRebaser, _stopwatch.ElapsedMilliseconds, LeapC.GetNow()); if (result != eLeapRS.eLeapRS_Success) { Debug.LogWarning("UpdateRebase call failed"); } } #endif if (_useInterpolation) { #if !UNITY_ANDROID || UNITY_EDITOR _smoothedTrackingLatency.value = Mathf.Min(_smoothedTrackingLatency.value, 30000f); _smoothedTrackingLatency.Update((float)(_leapController.Now() - _leapController.FrameTimestamp()), Time.deltaTime); #endif long timestamp = CalculateInterpolationTime() + (ExtrapolationAmount * 1000); _unityToLeapOffset = timestamp - (long)(Time.time * S_TO_NS); _leapController.GetInterpolatedFrameFromTime(_untransformedUpdateFrame, timestamp, CalculateInterpolationTime() - (BounceAmount * 1000)); } else { _leapController.Frame(_untransformedUpdateFrame); } if (_untransformedUpdateFrame != null) { transformFrame(_untransformedUpdateFrame, _transformedUpdateFrame); DispatchUpdateFrameEvent(_transformedUpdateFrame); } } protected virtual void FixedUpdate() { if (_frameOptimization == FrameOptimizationMode.ReuseUpdateForPhysics) { DispatchFixedFrameEvent(_transformedUpdateFrame); return; } if (_useInterpolation) { long timestamp; switch (_frameOptimization) { case FrameOptimizationMode.None: // By default we use Time.fixedTime to ensure that our hands are on the same // timeline as Update. We add an extrapolation value to help compensate // for latency. float extrapolatedTime = Time.fixedTime + CalculatePhysicsExtrapolation(); timestamp = (long)(extrapolatedTime * S_TO_NS) + _unityToLeapOffset; break; case FrameOptimizationMode.ReusePhysicsForUpdate: // If we are re-using physics frames for update, we don't even want to care // about Time.fixedTime, just grab the most recent interpolated timestamp // like we are in Update. timestamp = CalculateInterpolationTime() + (ExtrapolationAmount * 1000); break; default: throw new System.InvalidOperationException( "Unexpected frame optimization mode: " + _frameOptimization); } _leapController.GetInterpolatedFrame(_untransformedFixedFrame, timestamp); } else { _leapController.Frame(_untransformedFixedFrame); } if (_untransformedFixedFrame != null) { transformFrame(_untransformedFixedFrame, _transformedFixedFrame); DispatchFixedFrameEvent(_transformedFixedFrame); } } protected virtual void OnDestroy() { destroyController(); _isDestroyed = true; } protected virtual void OnApplicationFocus(bool hasFocus) { #if UNITY_ANDROID if (hasFocus) { CreateAndroidBinding(); } #endif } protected virtual void OnApplicationPause(bool isPaused) { #if UNITY_ANDROID if (_leapController != null) { if (isPaused) { _serviceBinder.Call("unbind"); } } #endif if (_leapController != null) { if (isPaused) { _leapController.StopConnection(); } else { _leapController.StartConnection(); } } } protected virtual void OnApplicationQuit() { destroyController(); _isDestroyed = true; } public float CalculatePhysicsExtrapolation() { switch (_physicsExtrapolation) { case PhysicsExtrapolationMode.None: return 0; case PhysicsExtrapolationMode.Auto: return Time.fixedDeltaTime; case PhysicsExtrapolationMode.Manual: return _physicsExtrapolationTime; default: throw new System.InvalidOperationException( "Unexpected physics extrapolation mode: " + _physicsExtrapolation); } } #endregion #region Public API /// /// Returns the Leap Controller instance. /// public Controller GetLeapController() { #if UNITY_EDITOR // Null check to deal with hot reloading. if (!_isDestroyed && _leapController == null) { createController(); } #endif return _leapController; } /// /// Returns true if the Leap Motion hardware is plugged in and this application is /// connected to the Leap Motion service. /// public bool IsConnected() { return GetLeapController().IsConnected; } /// /// Retransforms hand data from Leap space to the space of the Unity transform. /// This is only necessary if you're moving the LeapServiceProvider around in a /// custom script and trying to access Hand data from it directly afterward. /// public void RetransformFrames() { transformFrame(_untransformedUpdateFrame, _transformedUpdateFrame); transformFrame(_untransformedFixedFrame, _transformedFixedFrame); } /// /// Copies property settings from this LeapServiceProvider to the target /// LeapXRServiceProvider where applicable. Does not modify any XR-specific settings /// that only exist on the LeapXRServiceProvider. /// public void CopySettingsToLeapXRServiceProvider(LeapXRServiceProvider leapXRServiceProvider) { leapXRServiceProvider._interactionVolumeVisualization = _interactionVolumeVisualization; leapXRServiceProvider._frameOptimization = _frameOptimization; leapXRServiceProvider._physicsExtrapolation = _physicsExtrapolation; leapXRServiceProvider._physicsExtrapolationTime = _physicsExtrapolationTime; leapXRServiceProvider._workerThreadProfiling = _workerThreadProfiling; } /// /// Triggers a coroutine that sets appropriate policy flags and wait for them to be set to ensure we've changed mode /// /// Tracking mode to set public void ChangeTrackingMode(TrackingOptimizationMode trackingMode) { _trackingOptimization = trackingMode; if (_leapController == null) return; switch (trackingMode) { case TrackingOptimizationMode.Desktop: _leapController.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_SCREENTOP); _leapController.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD); break; case TrackingOptimizationMode.Screentop: _leapController.SetPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_SCREENTOP); _leapController.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD); break; case TrackingOptimizationMode.HMD: _leapController.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_SCREENTOP); _leapController.SetPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD); break; } } /// /// Gets the current mode by polling policy flags /// public TrackingOptimizationMode GetTrackingMode() { if (_leapController == null) return _trackingOptimization; var screenTopPolicySet = _leapController.IsPolicySet(Controller.PolicyFlag.POLICY_OPTIMIZE_SCREENTOP); var headMountedPolicySet = _leapController.IsPolicySet(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD); var desktopMode = !screenTopPolicySet && !headMountedPolicySet; if (desktopMode) { return TrackingOptimizationMode.Desktop; } var headMountedMode = !screenTopPolicySet && headMountedPolicySet; if (headMountedMode) { return TrackingOptimizationMode.HMD; } var screenTopMode = screenTopPolicySet && !headMountedPolicySet; if (screenTopMode) { return TrackingOptimizationMode.Screentop; } throw new Exception("Unknown tracking optimization mode"); } #endregion #region Internal Methods protected virtual long CalculateInterpolationTime(bool endOfFrame = false) { #if UNITY_ANDROID && !UNITY_EDITOR return _leapController.Now() - 16000; #else if (_leapController != null) { return _leapController.Now() - (long)_smoothedTrackingLatency.value; } else { return 0; } #endif } /// /// Initializes the policy flags. /// protected virtual void initializeFlags() { if (_preventInitializingTrackingMode) return; ChangeTrackingMode(_trackingOptimization); } /// /// Creates an instance of a Controller, initializing its policy flags and /// subscribing to its connection event. /// protected void createController() { #if SVR var bindStatus = CreateAndroidBinding(); if (!bindStatus) return; InitClockRebaser(); #endif if (_leapController != null) { return; } _leapController = new Controller(0, _serverNameSpace); _leapController.Device += (s, e) => { if (_onDeviceSafe != null) { _onDeviceSafe(e.Device); } }; if (_leapController.IsConnected) { initializeFlags(); } else { _leapController.Device += onHandControllerConnect; } if (_workerThreadProfiling) { //A controller will report profiling statistics for the duration of it's lifetime //so these events will never be unsubscribed from. _leapController.EndProfilingBlock += LeapProfiling.EndProfilingBlock; _leapController.BeginProfilingBlock += LeapProfiling.BeginProfilingBlock; _leapController.EndProfilingForThread += LeapProfiling.EndProfilingForThread; _leapController.BeginProfilingForThread += LeapProfiling.BeginProfilingForThread; } } /// /// Stops the connection for the existing instance of a Controller, clearing old /// policy flags and resetting the Controller to null. /// public void destroyController() { if (_leapController != null) { _leapController.StopConnection(); _leapController.Dispose(); _leapController = null; #if SVR if (_clockRebaser != IntPtr.Zero) { LeapC.DestroyClockRebaser(_clockRebaser); _stopwatch.Stop(); } #endif } } private int _framesSinceServiceConnectionChecked = 0; private int _numberOfReconnectionAttempts = 0; /// /// Checks whether this provider is connected to a service; /// If it is not, attempt to reconnect at regular intervals /// for MAX_RECONNECTION_ATTEMPTS /// protected bool checkConnectionIntegrity() { if (_leapController != null && _leapController.IsServiceConnected) { _framesSinceServiceConnectionChecked = 0; _numberOfReconnectionAttempts = 0; return true; } else if (_numberOfReconnectionAttempts < MAX_RECONNECTION_ATTEMPTS) { _framesSinceServiceConnectionChecked++; if (_framesSinceServiceConnectionChecked > RECONNECTION_INTERVAL) { _framesSinceServiceConnectionChecked = 0; _numberOfReconnectionAttempts++; Debug.LogWarning("Leap Service not connected; attempting to reconnect for try " + _numberOfReconnectionAttempts + "/" + MAX_RECONNECTION_ATTEMPTS + "...", this); using (new ProfilerSample("Reconnection Attempt")) { destroyController(); createController(); } } } return false; } protected void onHandControllerConnect(object sender, LeapEventArgs args) { initializeFlags(); #if SVR InitClockRebaser(); #endif if (_leapController != null) { _leapController.Device -= onHandControllerConnect; } } protected virtual void transformFrame(Frame source, Frame dest) { dest.CopyFrom(source).Transform(transform.GetLeapMatrix()); } #if SVR private void InitClockRebaser() { _stopwatch.Start(); eLeapRS result = LeapC.CreateClockRebaser(out _clockRebaser); if (result != eLeapRS.eLeapRS_Success) { Debug.LogError("Failed to create clock rebaser"); } if (_clockRebaser == IntPtr.Zero) { Debug.LogError("Clock rebaser is null"); } } #endif #endregion } }