/****************************************************************************** * Copyright (C) Ultraleap, Inc. 2011-2021. * * * * Use subject to the terms of the Apache License 2.0 available at * * http://www.apache.org/licenses/LICENSE-2.0, or another agreement * * between Ultraleap and you, your company or other organization. * ******************************************************************************/ using Leap.Unity.Attributes; using LeapInternal; using System; using UnityEngine; #if UNITY_2019_1_OR_NEWER using UnityEngine.Rendering; #endif namespace Leap.Unity { /// /// The LeapXRServiceProvider expands on the standard LeapServiceProvider to /// account for the offset of the Leap device with respect to the attached HMD and /// warp tracked hand positions based on the motion of the headset to account for the /// differing latencies of the two tracking systems. /// public class LeapXRServiceProvider : LeapServiceProvider { #region Inspector // Manual Device Offset #if UNITY_ANDROID private const float DEFAULT_DEVICE_OFFSET_Y_AXIS = -0.0114f; private const float DEFAULT_DEVICE_OFFSET_Z_AXIS = 0.0981f; private const float DEFAULT_DEVICE_TILT_X_AXIS = 0f; #else private const float DEFAULT_DEVICE_OFFSET_Y_AXIS = 0f; private const float DEFAULT_DEVICE_OFFSET_Z_AXIS = 0.12f; private const float DEFAULT_DEVICE_TILT_X_AXIS = 5f; #endif #if SVR private enum TimewarpMode { Default, Experimental_XR2 } private TimewarpMode _xr2TimewarpMode = TimewarpMode.Default; #endif public enum DeviceOffsetMode { Default, ManualHeadOffset, Transform } [Tooltip("Allow manual adjustment of the Leap device's virtual offset and tilt. These " + "settings can be used to match the physical position and orientation of the " + "Leap Motion sensor on a tracked device it is mounted on (such as a VR " + "headset). ")] [SerializeField, OnEditorChange("deviceOffsetMode")] private DeviceOffsetMode _deviceOffsetMode; public DeviceOffsetMode deviceOffsetMode { get { return _deviceOffsetMode; } set { _deviceOffsetMode = value; if (_deviceOffsetMode == DeviceOffsetMode.Default) { deviceOffsetYAxis = DEFAULT_DEVICE_OFFSET_Y_AXIS; deviceOffsetZAxis = DEFAULT_DEVICE_OFFSET_Z_AXIS; deviceTiltXAxis = DEFAULT_DEVICE_TILT_X_AXIS; } } } [Tooltip("Adjusts the Leap Motion device's virtual height offset from the tracked " + "headset position. This should match the vertical offset of the physical " + "device with respect to the headset in meters.")] [SerializeField] [Range(-0.50F, 0.50F)] private float _deviceOffsetYAxis = DEFAULT_DEVICE_OFFSET_Y_AXIS; public float deviceOffsetYAxis { get { return _deviceOffsetYAxis; } set { _deviceOffsetYAxis = value; } } [Tooltip("Adjusts the Leap Motion device's virtual depth offset from the tracked " + "headset position. This should match the forward offset of the physical " + "device with respect to the headset in meters.")] [SerializeField] [Range(-0.50F, 0.50F)] private float _deviceOffsetZAxis = DEFAULT_DEVICE_OFFSET_Z_AXIS; public float deviceOffsetZAxis { get { return _deviceOffsetZAxis; } set { _deviceOffsetZAxis = value; } } [Tooltip("Adjusts the Leap Motion device's virtual X axis tilt. This should match " + "the tilt of the physical device with respect to the headset in degrees.")] [SerializeField] [Range(-90.0F, 90.0F)] private float _deviceTiltXAxis = DEFAULT_DEVICE_TILT_X_AXIS; public float deviceTiltXAxis { get { return _deviceTiltXAxis; } set { _deviceTiltXAxis = value; } } [Tooltip("Allows for the manual placement of the Leap Tracking Device." + "This device offset mode is incompatible with Temporal Warping.")] [SerializeField] private Transform _deviceOrigin; public Transform deviceOrigin { get { return _deviceOrigin; } set { _deviceOrigin = value; } } [Tooltip("Specifies the main camera. Required for XR2 based platforms. " + "Falls back to Camera.main if not set")] [SerializeField] private Camera _mainCamera; // Redundant backing field, used to present value in editor at parent level public Camera mainCamera { get { return MainCameraProvider.mainCamera; } set { // Not everything accesses the main camera via the property, so to be safe we also set the backing field _mainCamera = value; MainCameraProvider.mainCamera = value; } } // Temporal Warping #if UNITY_STANDALONE private const int DEFAULT_WARP_ADJUSTMENT = 17; #elif SVR private const int DEFAULT_WARP_ADJUSTMENT = 35; // Tuned for XR2 on a Morpheus SKU3 #else private const int DEFAULT_WARP_ADJUSTMENT = 17; #endif public enum TemporalWarpingMode { Auto, Manual, Images, Off } [Tooltip("Temporal warping prevents the hand coordinate system from 'swimming' or " + "'bouncing' when the headset moves and the user's hands stay still. " + "This phenomenon is caused by the differing amounts of latencies inherent " + "in the two systems. " + "For PC VR and Android VR, temporal warping should set to 'Auto', as the " + "correct value can be chosen automatically for these platforms. " + "Some non-standard platforms may use 'Manual' mode to adjust their " + "latency compensation amount for temporal warping. " + "Use 'Images' for scenarios that overlay Leap device images on tracked " + "hand data.")] [SerializeField] private TemporalWarpingMode _temporalWarpingMode = TemporalWarpingMode.Auto; /// /// The time in milliseconds between the current frame's headset position and the /// time at which the Leap frame was captured. /// [Tooltip("The time in milliseconds between the current frame's headset position and " + "the time at which the Leap frame was captured.")] [SerializeField] private int _customWarpAdjustment = DEFAULT_WARP_ADJUSTMENT; public int warpingAdjustment { get { if (_temporalWarpingMode == TemporalWarpingMode.Manual) { return _customWarpAdjustment; } else { return DEFAULT_WARP_ADJUSTMENT; } } set { if (_temporalWarpingMode != TemporalWarpingMode.Manual) { Debug.LogWarning("Setting custom warping adjustment amount, but the " + "temporal warping mode is not Manual, so the value will be " + "ignored."); } _customWarpAdjustment = value; } } // Pre-cull Latching [Tooltip("Pass updated transform matrices to hands with materials that utilize the " + "VertexOffsetShader. Won't have any effect on hands that don't take into " + "account shader-global vertex offsets in their material shaders.")] [SerializeField] protected bool _updateHandInPrecull = false; public bool updateHandInPrecull { get { return _updateHandInPrecull; } set { resetShaderTransforms(); _updateHandInPrecull = value; } } #endregion #region Internal Memory protected TransformHistory transformHistory = new TransformHistory(); protected bool manualUpdateHasBeenCalledSinceUpdate; protected Vector3 warpedPosition = Vector3.zero; protected Quaternion warpedRotation = Quaternion.identity; protected Matrix4x4[] _transformArray = new Matrix4x4[2]; private Pose? _trackingBaseDeltaPose = null; [NonSerialized] public long imageTimeStamp = 0; #endregion #region Unity Events protected override void Reset() { base.Reset(); editTimePose = TestHandFactory.TestHandPose.HeadMountedB; _interactionVolumeVisualization = InteractionVolumeVisualization.Automatic; _mainCamera = MainCameraProvider.mainCamera; if (_mainCamera != null) { Debug.Log("Camera.Main automatically assigned"); } } private void OnValidate() { this.transform.hideFlags = HideFlags.NotEditable; } protected override void OnEnable() { resetShaderTransforms(); // Assign the main camera if it looks like one is available and it's not yet been set on the backing field // NB this may be the case if the provider is created via AddComponent, as in MRTK if (_mainCamera == null && MainCameraProvider.mainCamera != null) { _mainCamera = MainCameraProvider.mainCamera; } #if XR_LEGACY_INPUT_AVAILABLE if (_mainCamera.GetComponent() == null) { _mainCamera.gameObject.AddComponent().UseRelativeTransform = true; } #endif #if UNITY_2019_1_OR_NEWER if (GraphicsSettings.renderPipelineAsset != null) { RenderPipelineManager.beginCameraRendering -= onBeginRendering; RenderPipelineManager.beginCameraRendering += onBeginRendering; } else { Camera.onPreCull -= onPreCull; // No multiple-subscription. Camera.onPreCull += onPreCull; } #else Camera.onPreCull -= onPreCull; // No multiple-subscription. Camera.onPreCull += onPreCull; #endif #if UNITY_ANDROID base.OnEnable(); #endif } protected override void OnDisable() { resetShaderTransforms(); #if UNITY_2019_1_OR_NEWER if (GraphicsSettings.renderPipelineAsset != null) { RenderPipelineManager.beginCameraRendering -= onBeginRendering; } else { Camera.onPreCull -= onPreCull; // No multiple-subscription. } #else Camera.onPreCull -= onPreCull; // No multiple-subscription. #endif #if UNITY_ANDROID base.OnDisable(); #endif } protected override void Start() { base.Start(); if (_deviceOffsetMode == DeviceOffsetMode.Transform && _deviceOrigin == null) { Debug.LogError("Cannot use the Transform device offset mode without " + "specifying a Transform to use as the device origin.", this); _deviceOffsetMode = DeviceOffsetMode.Default; } if (Application.isPlaying && _mainCamera == null && _temporalWarpingMode != TemporalWarpingMode.Off) { Debug.LogError("Cannot perform temporal warping with no pre-cull camera."); } //Get the local tracked pose from the XR Headset so we can calculate the _trackingBaseDeltaPose from it var trackedPose = new Pose(XRSupportUtil.GetXRNodeCenterEyeLocalPosition(), XRSupportUtil.GetXRNodeCenterEyeLocalRotation()); //Find the pose delta from the "local" tracked pose to the actual camera pose, we will use this later on to maintain offsets if (!_trackingBaseDeltaPose.HasValue) { _trackingBaseDeltaPose = _mainCamera.transform.ToLocalPose().mul(trackedPose.inverse()); } } protected override void Update() { manualUpdateHasBeenCalledSinceUpdate = false; base.Update(); if (_leapController != null) { imageTimeStamp = _leapController.FrameTimestamp(); } } void LateUpdate() { var projectionMatrix = _mainCamera == null ? Matrix4x4.identity : _mainCamera.projectionMatrix; switch (SystemInfo.graphicsDeviceType) { #if !UNITY_2017_2_OR_NEWER case UnityEngine.Rendering.GraphicsDeviceType.Direct3D9: #endif case UnityEngine.Rendering.GraphicsDeviceType.Direct3D11: case UnityEngine.Rendering.GraphicsDeviceType.Direct3D12: for (int i = 0; i < 4; i++) { projectionMatrix[1, i] = -projectionMatrix[1, i]; } // Scale and bias from OpenGL -> D3D depth range for (int i = 0; i < 4; i++) { projectionMatrix[2, i] = projectionMatrix[2, i] * 0.5f + projectionMatrix[3, i] * 0.5f; } break; } // Update Image Warping Vector3 pastPosition; Quaternion pastRotation; transformHistory.SampleTransform(imageTimeStamp - (long)(warpingAdjustment * 1000f), out pastPosition, out pastRotation); // Use _tweenImageWarping var currCenterRotation = XRSupportUtil.GetXRNodeCenterEyeLocalRotation(); var imageReferenceRotation = _temporalWarpingMode != TemporalWarpingMode.Off ? pastRotation : currCenterRotation; Quaternion imageQuatWarp = Quaternion.Inverse(currCenterRotation) * imageReferenceRotation; imageQuatWarp = Quaternion.Euler(imageQuatWarp.eulerAngles.x, imageQuatWarp.eulerAngles.y, -imageQuatWarp.eulerAngles.z); Matrix4x4 imageMatWarp = projectionMatrix #if UNITY_2019_2_OR_NEWER // The camera projection matrices seem to have vertically inverted... * Matrix4x4.TRS(Vector3.zero, imageQuatWarp, new Vector3(1f, -1f, 1f)) #else * Matrix4x4.TRS(Vector3.zero, imageQuatWarp, Vector3.one) #endif * projectionMatrix.inverse; Shader.SetGlobalMatrix("_LeapGlobalWarpedOffset", imageMatWarp); } #if UNITY_2019_1_OR_NEWER protected virtual void onBeginRendering(ScriptableRenderContext context, Camera camera) { onPreCull(camera); } #endif protected virtual void onPreCull(Camera preCullingCamera) { if (preCullingCamera != _mainCamera) { return; } #if UNITY_EDITOR if (!Application.isPlaying) { return; } #endif if (_mainCamera == null || _leapController == null) { if (_temporalWarpingMode == TemporalWarpingMode.Auto || _temporalWarpingMode == TemporalWarpingMode.Manual) { Debug.LogError("The camera or controller need to be set for temporal warping to work"); } return; } Pose trackedPose; if (_deviceOffsetMode == DeviceOffsetMode.Default || _deviceOffsetMode == DeviceOffsetMode.ManualHeadOffset) { //Get the local tracked pose from the XR Headset trackedPose = new Pose(XRSupportUtil.GetXRNodeCenterEyeLocalPosition(), XRSupportUtil.GetXRNodeCenterEyeLocalRotation()); //Use the _trackingBaseDeltaPose calculated on start to convert the local spaced trackedPose into a world space position trackedPose = _trackingBaseDeltaPose.Value.mul(trackedPose); } else if (_deviceOffsetMode == DeviceOffsetMode.Transform) { trackedPose = deviceOrigin.ToPose(); } else { Debug.LogError($"Unsupported DeviceOffsetMode: {_deviceOffsetMode}"); return; } transformHistory.UpdateDelay(trackedPose, _leapController.Now()); OnPreCullHandTransforms(_mainCamera); } #endregion #region LeapServiceProvider Overrides protected override long CalculateInterpolationTime(bool endOfFrame = false) { if (_leapController == null) { return 0; } #if SVR if (_xr2TimewarpMode == TimewarpMode.Experimental_XR2) { return GetPredictedDisplayTime_LeapTime(); } else { return _leapController.Now() - 16000; } #elif UNITY_ANDROID return _leapController.Now() - 16000; #else return _leapController.Now() - (long)_smoothedTrackingLatency.value + ((updateHandInPrecull && !endOfFrame) ? (long)(Time.smoothDeltaTime * S_TO_NS / Time.timeScale) : 0); #endif } /// /// Initializes the policy flags. /// The POLICY_OPTIMIZE_HMD flag improves tracking for head-mounted devices. /// protected override void initializeFlags() { ChangeTrackingMode(TrackingOptimizationMode.HMD); } protected override void transformFrame(Frame source, Frame dest) { LeapTransform leapTransform = LeapTransform.Identity; if (mainCamera != null) { //By default, use the camera transform matrix to transform the frame into leapTransform = mainCamera.transform.GetLeapMatrix(); leapTransform.scale = Vector.Ones * 1e-3f; //If the application is playing then we can try to use temporal warping if (Application.isPlaying) { leapTransform = GetWarpedMatrix(source.Timestamp); } } dest.CopyFrom(source).Transform(leapTransform); } #endregion #region Internal Methods /// /// Resets shader globals for the Hand transforms. /// protected void resetShaderTransforms() { _transformArray[0] = Matrix4x4.identity; _transformArray[1] = Matrix4x4.identity; Shader.SetGlobalMatrixArray(HAND_ARRAY_GLOBAL_NAME, _transformArray); } protected virtual LeapTransform GetWarpedMatrix(long timestamp, bool updateTemporalCompensation = true) { if (_mainCamera == null || this == null) { return LeapTransform.Identity; } LeapTransform leapTransform = new LeapTransform(); // If temporal warping is turned off if (_temporalWarpingMode == TemporalWarpingMode.Off) { //Calculate the Current Pose Pose currentPose = Pose.identity; if (_deviceOffsetMode == DeviceOffsetMode.Transform && deviceOrigin != null) { currentPose = deviceOrigin.ToPose(); } else { transformHistory.SampleTransform(timestamp, out currentPose.position, out currentPose.rotation); } warpedPosition = currentPose.position; warpedRotation = currentPose.rotation; } //Calculate a Temporally Warped Pose else if (updateTemporalCompensation && transformHistory.history.IsFull) { #if SVR if (_xr2TimewarpMode == TimewarpMode.Default && transformHistory.history.IsFull) { var imageAdjustment = _temporalWarpingMode == TemporalWarpingMode.Images ? -20000 : 0; var sampleTimestamp = timestamp - (long)(warpingAdjustment * 1000f) - imageAdjustment; transformHistory.SampleTransform(sampleTimestamp, out warpedPosition, out warpedRotation); } else if (_xr2TimewarpMode == TimewarpMode.Experimental_XR2) { // Get the predicted display time for the current frame in milliseconds, then get the predicted head pose float predictedDisplayTime_ms = SxrShim.GetPredictedDisplayTime(SystemInfo.graphicsMultiThreaded); Vector3 predictedWarpedPosition; Quaternion predictedWarpedRotation; SxrShim.GetPredictedHeadPose(predictedDisplayTime_ms, out predictedWarpedRotation, out predictedWarpedPosition); warpedPosition.x = -predictedWarpedPosition.x; warpedPosition.y = -predictedWarpedPosition.y; warpedPosition.z = predictedWarpedPosition.z; warpedRotation = predictedWarpedRotation; } #else if (transformHistory.history.IsFull) { var imageAdjustment = _temporalWarpingMode == TemporalWarpingMode.Images ? -20000 : 0; var sampleTimestamp = timestamp - (long)(warpingAdjustment * 1000f) - imageAdjustment; transformHistory.SampleTransform(sampleTimestamp, out warpedPosition, out warpedRotation); } #endif } // Normalize the rotation Quaternion. warpedRotation = warpedRotation.ToNormalized(); // If we are NOT using a transform to offset the tracking if (_deviceOffsetMode != DeviceOffsetMode.Transform) { warpedPosition += warpedRotation * Vector3.up * deviceOffsetYAxis + warpedRotation * Vector3.forward * deviceOffsetZAxis; warpedRotation *= Quaternion.Euler(deviceTiltXAxis, 0f, 0f); warpedRotation *= Quaternion.Euler(-90f, 180f, 0f); } else { warpedRotation *= Quaternion.Euler(-90f, 90f, 90f); } #if !SVR // Use the _mainCamera parent to transfrom the warped positions so the player can move around if (_mainCamera.transform.parent != null) { leapTransform = new LeapTransform( _mainCamera.transform.parent.TransformPoint(warpedPosition).ToVector(), _mainCamera.transform.parent.TransformRotation(warpedRotation).ToLeapQuaternion(), Vector.Ones * 1e-3f ); } else #endif { leapTransform = new LeapTransform( warpedPosition.ToVector(), warpedRotation.ToLeapQuaternion(), Vector.Ones * 1e-3f ); } leapTransform.MirrorZ(); return leapTransform; } protected void transformHands(ref LeapTransform LeftHand, ref LeapTransform RightHand) { LeapTransform leapTransform = GetWarpedMatrix(0, false); LeftHand = new LeapTransform(leapTransform.TransformPoint(LeftHand.translation), leapTransform.TransformQuaternion(LeftHand.rotation)); RightHand = new LeapTransform(leapTransform.TransformPoint(RightHand.translation), leapTransform.TransformQuaternion(RightHand.rotation)); } protected void OnPreCullHandTransforms(Camera camera) { if (updateHandInPrecull) { //Don't update pre cull for preview, reflection, or scene view cameras if (camera == null) { camera = _mainCamera; } switch (camera.cameraType) { case CameraType.Preview: #if UNITY_2017_1_OR_NEWER case CameraType.Reflection: #endif case CameraType.SceneView: return; } if (Application.isPlaying && !manualUpdateHasBeenCalledSinceUpdate && _leapController != null) { manualUpdateHasBeenCalledSinceUpdate = true; //Find the left and/or right hand(s) to latch Hand leftHand = null, rightHand = null; LeapTransform precullLeftHand = LeapTransform.Identity; LeapTransform precullRightHand = LeapTransform.Identity; for (int i = 0; i < CurrentFrame.Hands.Count; i++) { Hand updateHand = CurrentFrame.Hands[i]; if (updateHand.IsLeft && leftHand == null) { leftHand = updateHand; } else if (updateHand.IsRight && rightHand == null) { rightHand = updateHand; } } //Determine their new Transforms var interpolationTime = CalculateInterpolationTime(); _leapController.GetInterpolatedLeftRightTransform( interpolationTime + (ExtrapolationAmount * 1000), interpolationTime - (BounceAmount * 1000), (leftHand != null ? leftHand.Id : 0), (rightHand != null ? rightHand.Id : 0), out precullLeftHand, out precullRightHand); bool leftValid = precullLeftHand.translation != Vector.Zero; bool rightValid = precullRightHand.translation != Vector.Zero; transformHands(ref precullLeftHand, ref precullRightHand); //Calculate the delta Transforms if (rightHand != null && rightValid) { _transformArray[0] = Matrix4x4.TRS(precullRightHand.translation.ToVector3(), precullRightHand.rotation.ToQuaternion(), Vector3.one) * Matrix4x4.Inverse(Matrix4x4.TRS(rightHand.PalmPosition.ToVector3(), rightHand.Rotation.ToQuaternion(), Vector3.one)); } if (leftHand != null && leftValid) { _transformArray[1] = Matrix4x4.TRS(precullLeftHand.translation.ToVector3(), precullLeftHand.rotation.ToQuaternion(), Vector3.one) * Matrix4x4.Inverse(Matrix4x4.TRS(leftHand.PalmPosition.ToVector3(), leftHand.Rotation.ToQuaternion(), Vector3.one)); } //Apply inside of the vertex shader Shader.SetGlobalMatrixArray(HAND_ARRAY_GLOBAL_NAME, _transformArray); } } } #if SVR /// /// Return the predicted display time as a leap time /// /// private long GetPredictedDisplayTime_LeapTime() { long leapClock = 0; // Predicted display time for the current frame in milliseconds float displayTime_ms = SxrShim.GetPredictedDisplayTime(SystemInfo.graphicsMultiThreaded); if (_clockRebaser != IntPtr.Zero) { LeapC.RebaseClock(_clockRebaser, (long)displayTime_ms + _stopwatch.ElapsedMilliseconds, out leapClock); } return leapClock; } #endif #endregion } }