/******************************************************************************
* Copyright (C) Ultraleap, Inc. 2011-2021. *
* *
* Use subject to the terms of the Apache License 2.0 available at *
* http://www.apache.org/licenses/LICENSE-2.0, or another agreement *
* between Ultraleap and you, your company or other organization. *
******************************************************************************/
using Leap.Unity.Attributes;
using LeapInternal;
using System;
using UnityEngine;
#if UNITY_2019_1_OR_NEWER
using UnityEngine.Rendering;
#endif
namespace Leap.Unity
{
///
/// The LeapXRServiceProvider expands on the standard LeapServiceProvider to
/// account for the offset of the Leap device with respect to the attached HMD and
/// warp tracked hand positions based on the motion of the headset to account for the
/// differing latencies of the two tracking systems.
///
public class LeapXRServiceProvider : LeapServiceProvider
{
#region Inspector
// Manual Device Offset
#if UNITY_ANDROID
private const float DEFAULT_DEVICE_OFFSET_Y_AXIS = -0.0114f;
private const float DEFAULT_DEVICE_OFFSET_Z_AXIS = 0.0981f;
private const float DEFAULT_DEVICE_TILT_X_AXIS = 0f;
#else
private const float DEFAULT_DEVICE_OFFSET_Y_AXIS = 0f;
private const float DEFAULT_DEVICE_OFFSET_Z_AXIS = 0.12f;
private const float DEFAULT_DEVICE_TILT_X_AXIS = 5f;
#endif
#if SVR
private enum TimewarpMode
{
Default,
Experimental_XR2
}
private TimewarpMode _xr2TimewarpMode = TimewarpMode.Default;
#endif
public enum DeviceOffsetMode
{
Default,
ManualHeadOffset,
Transform
}
[Tooltip("Allow manual adjustment of the Leap device's virtual offset and tilt. These "
+ "settings can be used to match the physical position and orientation of the "
+ "Leap Motion sensor on a tracked device it is mounted on (such as a VR "
+ "headset). ")]
[SerializeField, OnEditorChange("deviceOffsetMode")]
private DeviceOffsetMode _deviceOffsetMode;
public DeviceOffsetMode deviceOffsetMode
{
get { return _deviceOffsetMode; }
set
{
_deviceOffsetMode = value;
if (_deviceOffsetMode == DeviceOffsetMode.Default)
{
deviceOffsetYAxis = DEFAULT_DEVICE_OFFSET_Y_AXIS;
deviceOffsetZAxis = DEFAULT_DEVICE_OFFSET_Z_AXIS;
deviceTiltXAxis = DEFAULT_DEVICE_TILT_X_AXIS;
}
}
}
[Tooltip("Adjusts the Leap Motion device's virtual height offset from the tracked "
+ "headset position. This should match the vertical offset of the physical "
+ "device with respect to the headset in meters.")]
[SerializeField]
[Range(-0.50F, 0.50F)]
private float _deviceOffsetYAxis = DEFAULT_DEVICE_OFFSET_Y_AXIS;
public float deviceOffsetYAxis
{
get { return _deviceOffsetYAxis; }
set { _deviceOffsetYAxis = value; }
}
[Tooltip("Adjusts the Leap Motion device's virtual depth offset from the tracked "
+ "headset position. This should match the forward offset of the physical "
+ "device with respect to the headset in meters.")]
[SerializeField]
[Range(-0.50F, 0.50F)]
private float _deviceOffsetZAxis = DEFAULT_DEVICE_OFFSET_Z_AXIS;
public float deviceOffsetZAxis
{
get { return _deviceOffsetZAxis; }
set { _deviceOffsetZAxis = value; }
}
[Tooltip("Adjusts the Leap Motion device's virtual X axis tilt. This should match "
+ "the tilt of the physical device with respect to the headset in degrees.")]
[SerializeField]
[Range(-90.0F, 90.0F)]
private float _deviceTiltXAxis = DEFAULT_DEVICE_TILT_X_AXIS;
public float deviceTiltXAxis
{
get { return _deviceTiltXAxis; }
set { _deviceTiltXAxis = value; }
}
[Tooltip("Allows for the manual placement of the Leap Tracking Device." +
"This device offset mode is incompatible with Temporal Warping.")]
[SerializeField]
private Transform _deviceOrigin;
public Transform deviceOrigin
{
get { return _deviceOrigin; }
set { _deviceOrigin = value; }
}
[Tooltip("Specifies the main camera. Required for XR2 based platforms. "
+ "Falls back to Camera.main if not set")]
[SerializeField]
private Camera _mainCamera; // Redundant backing field, used to present value in editor at parent level
public Camera mainCamera
{
get
{
return MainCameraProvider.mainCamera;
}
set
{
// Not everything accesses the main camera via the property, so to be safe we also set the backing field
_mainCamera = value;
MainCameraProvider.mainCamera = value;
}
}
// Temporal Warping
#if UNITY_STANDALONE
private const int DEFAULT_WARP_ADJUSTMENT = 17;
#elif SVR
private const int DEFAULT_WARP_ADJUSTMENT = 35; // Tuned for XR2 on a Morpheus SKU3
#else
private const int DEFAULT_WARP_ADJUSTMENT = 17;
#endif
public enum TemporalWarpingMode
{
Auto,
Manual,
Images,
Off
}
[Tooltip("Temporal warping prevents the hand coordinate system from 'swimming' or "
+ "'bouncing' when the headset moves and the user's hands stay still. "
+ "This phenomenon is caused by the differing amounts of latencies inherent "
+ "in the two systems. "
+ "For PC VR and Android VR, temporal warping should set to 'Auto', as the "
+ "correct value can be chosen automatically for these platforms. "
+ "Some non-standard platforms may use 'Manual' mode to adjust their "
+ "latency compensation amount for temporal warping. "
+ "Use 'Images' for scenarios that overlay Leap device images on tracked "
+ "hand data.")]
[SerializeField]
private TemporalWarpingMode _temporalWarpingMode = TemporalWarpingMode.Auto;
///
/// The time in milliseconds between the current frame's headset position and the
/// time at which the Leap frame was captured.
///
[Tooltip("The time in milliseconds between the current frame's headset position and "
+ "the time at which the Leap frame was captured.")]
[SerializeField]
private int _customWarpAdjustment = DEFAULT_WARP_ADJUSTMENT;
public int warpingAdjustment
{
get
{
if (_temporalWarpingMode == TemporalWarpingMode.Manual)
{
return _customWarpAdjustment;
}
else
{
return DEFAULT_WARP_ADJUSTMENT;
}
}
set
{
if (_temporalWarpingMode != TemporalWarpingMode.Manual)
{
Debug.LogWarning("Setting custom warping adjustment amount, but the "
+ "temporal warping mode is not Manual, so the value will be "
+ "ignored.");
}
_customWarpAdjustment = value;
}
}
// Pre-cull Latching
[Tooltip("Pass updated transform matrices to hands with materials that utilize the "
+ "VertexOffsetShader. Won't have any effect on hands that don't take into "
+ "account shader-global vertex offsets in their material shaders.")]
[SerializeField]
protected bool _updateHandInPrecull = false;
public bool updateHandInPrecull
{
get { return _updateHandInPrecull; }
set
{
resetShaderTransforms();
_updateHandInPrecull = value;
}
}
#endregion
#region Internal Memory
protected TransformHistory transformHistory = new TransformHistory();
protected bool manualUpdateHasBeenCalledSinceUpdate;
protected Vector3 warpedPosition = Vector3.zero;
protected Quaternion warpedRotation = Quaternion.identity;
protected Matrix4x4[] _transformArray = new Matrix4x4[2];
private Pose? _trackingBaseDeltaPose = null;
[NonSerialized]
public long imageTimeStamp = 0;
#endregion
#region Unity Events
protected override void Reset()
{
base.Reset();
editTimePose = TestHandFactory.TestHandPose.HeadMountedB;
_interactionVolumeVisualization = InteractionVolumeVisualization.Automatic;
_mainCamera = MainCameraProvider.mainCamera;
if (_mainCamera != null)
{
Debug.Log("Camera.Main automatically assigned");
}
}
private void OnValidate()
{
this.transform.hideFlags = HideFlags.NotEditable;
}
protected override void OnEnable()
{
resetShaderTransforms();
// Assign the main camera if it looks like one is available and it's not yet been set on the backing field
// NB this may be the case if the provider is created via AddComponent, as in MRTK
if (_mainCamera == null && MainCameraProvider.mainCamera != null)
{
_mainCamera = MainCameraProvider.mainCamera;
}
#if XR_LEGACY_INPUT_AVAILABLE
if (_mainCamera.GetComponent() == null)
{
_mainCamera.gameObject.AddComponent().UseRelativeTransform = true;
}
#endif
#if UNITY_2019_1_OR_NEWER
if (GraphicsSettings.renderPipelineAsset != null)
{
RenderPipelineManager.beginCameraRendering -= onBeginRendering;
RenderPipelineManager.beginCameraRendering += onBeginRendering;
}
else
{
Camera.onPreCull -= onPreCull; // No multiple-subscription.
Camera.onPreCull += onPreCull;
}
#else
Camera.onPreCull -= onPreCull; // No multiple-subscription.
Camera.onPreCull += onPreCull;
#endif
#if UNITY_ANDROID
base.OnEnable();
#endif
}
protected override void OnDisable()
{
resetShaderTransforms();
#if UNITY_2019_1_OR_NEWER
if (GraphicsSettings.renderPipelineAsset != null)
{
RenderPipelineManager.beginCameraRendering -= onBeginRendering;
}
else
{
Camera.onPreCull -= onPreCull; // No multiple-subscription.
}
#else
Camera.onPreCull -= onPreCull; // No multiple-subscription.
#endif
#if UNITY_ANDROID
base.OnDisable();
#endif
}
protected override void Start()
{
base.Start();
if (_deviceOffsetMode == DeviceOffsetMode.Transform && _deviceOrigin == null)
{
Debug.LogError("Cannot use the Transform device offset mode without " +
"specifying a Transform to use as the device origin.", this);
_deviceOffsetMode = DeviceOffsetMode.Default;
}
if (Application.isPlaying && _mainCamera == null && _temporalWarpingMode != TemporalWarpingMode.Off)
{
Debug.LogError("Cannot perform temporal warping with no pre-cull camera.");
}
//Get the local tracked pose from the XR Headset so we can calculate the _trackingBaseDeltaPose from it
var trackedPose = new Pose(XRSupportUtil.GetXRNodeCenterEyeLocalPosition(), XRSupportUtil.GetXRNodeCenterEyeLocalRotation());
//Find the pose delta from the "local" tracked pose to the actual camera pose, we will use this later on to maintain offsets
if (!_trackingBaseDeltaPose.HasValue)
{
_trackingBaseDeltaPose = _mainCamera.transform.ToLocalPose().mul(trackedPose.inverse());
}
}
protected override void Update()
{
manualUpdateHasBeenCalledSinceUpdate = false;
base.Update();
if (_leapController != null)
{
imageTimeStamp = _leapController.FrameTimestamp();
}
}
void LateUpdate()
{
var projectionMatrix = _mainCamera == null ? Matrix4x4.identity
: _mainCamera.projectionMatrix;
switch (SystemInfo.graphicsDeviceType)
{
#if !UNITY_2017_2_OR_NEWER
case UnityEngine.Rendering.GraphicsDeviceType.Direct3D9:
#endif
case UnityEngine.Rendering.GraphicsDeviceType.Direct3D11:
case UnityEngine.Rendering.GraphicsDeviceType.Direct3D12:
for (int i = 0; i < 4; i++)
{
projectionMatrix[1, i] = -projectionMatrix[1, i];
}
// Scale and bias from OpenGL -> D3D depth range
for (int i = 0; i < 4; i++)
{
projectionMatrix[2, i] = projectionMatrix[2, i] * 0.5f
+ projectionMatrix[3, i] * 0.5f;
}
break;
}
// Update Image Warping
Vector3 pastPosition; Quaternion pastRotation;
transformHistory.SampleTransform(imageTimeStamp
- (long)(warpingAdjustment * 1000f),
out pastPosition, out pastRotation);
// Use _tweenImageWarping
var currCenterRotation = XRSupportUtil.GetXRNodeCenterEyeLocalRotation();
var imageReferenceRotation = _temporalWarpingMode != TemporalWarpingMode.Off
? pastRotation
: currCenterRotation;
Quaternion imageQuatWarp = Quaternion.Inverse(currCenterRotation)
* imageReferenceRotation;
imageQuatWarp = Quaternion.Euler(imageQuatWarp.eulerAngles.x,
imageQuatWarp.eulerAngles.y,
-imageQuatWarp.eulerAngles.z);
Matrix4x4 imageMatWarp = projectionMatrix
#if UNITY_2019_2_OR_NEWER
// The camera projection matrices seem to have vertically inverted...
* Matrix4x4.TRS(Vector3.zero, imageQuatWarp, new Vector3(1f, -1f, 1f))
#else
* Matrix4x4.TRS(Vector3.zero, imageQuatWarp, Vector3.one)
#endif
* projectionMatrix.inverse;
Shader.SetGlobalMatrix("_LeapGlobalWarpedOffset", imageMatWarp);
}
#if UNITY_2019_1_OR_NEWER
protected virtual void onBeginRendering(ScriptableRenderContext context, Camera camera) { onPreCull(camera); }
#endif
protected virtual void onPreCull(Camera preCullingCamera)
{
if (preCullingCamera != _mainCamera)
{
return;
}
#if UNITY_EDITOR
if (!Application.isPlaying)
{
return;
}
#endif
if (_mainCamera == null || _leapController == null)
{
if (_temporalWarpingMode == TemporalWarpingMode.Auto || _temporalWarpingMode == TemporalWarpingMode.Manual)
{
Debug.LogError("The camera or controller need to be set for temporal warping to work");
}
return;
}
Pose trackedPose;
if (_deviceOffsetMode == DeviceOffsetMode.Default
|| _deviceOffsetMode == DeviceOffsetMode.ManualHeadOffset)
{
//Get the local tracked pose from the XR Headset
trackedPose = new Pose(XRSupportUtil.GetXRNodeCenterEyeLocalPosition(), XRSupportUtil.GetXRNodeCenterEyeLocalRotation());
//Use the _trackingBaseDeltaPose calculated on start to convert the local spaced trackedPose into a world space position
trackedPose = _trackingBaseDeltaPose.Value.mul(trackedPose);
}
else if (_deviceOffsetMode == DeviceOffsetMode.Transform)
{
trackedPose = deviceOrigin.ToPose();
}
else
{
Debug.LogError($"Unsupported DeviceOffsetMode: {_deviceOffsetMode}");
return;
}
transformHistory.UpdateDelay(trackedPose, _leapController.Now());
OnPreCullHandTransforms(_mainCamera);
}
#endregion
#region LeapServiceProvider Overrides
protected override long CalculateInterpolationTime(bool endOfFrame = false)
{
if (_leapController == null)
{
return 0;
}
#if SVR
if (_xr2TimewarpMode == TimewarpMode.Experimental_XR2)
{
return GetPredictedDisplayTime_LeapTime();
}
else
{
return _leapController.Now() - 16000;
}
#elif UNITY_ANDROID
return _leapController.Now() - 16000;
#else
return _leapController.Now()
- (long)_smoothedTrackingLatency.value
+ ((updateHandInPrecull && !endOfFrame) ?
(long)(Time.smoothDeltaTime * S_TO_NS / Time.timeScale)
: 0);
#endif
}
///
/// Initializes the policy flags.
/// The POLICY_OPTIMIZE_HMD flag improves tracking for head-mounted devices.
///
protected override void initializeFlags()
{
ChangeTrackingMode(TrackingOptimizationMode.HMD);
}
protected override void transformFrame(Frame source, Frame dest)
{
LeapTransform leapTransform = LeapTransform.Identity;
if (mainCamera != null)
{
//By default, use the camera transform matrix to transform the frame into
leapTransform = mainCamera.transform.GetLeapMatrix();
leapTransform.scale = Vector.Ones * 1e-3f;
//If the application is playing then we can try to use temporal warping
if (Application.isPlaying)
{
leapTransform = GetWarpedMatrix(source.Timestamp);
}
}
dest.CopyFrom(source).Transform(leapTransform);
}
#endregion
#region Internal Methods
///
/// Resets shader globals for the Hand transforms.
///
protected void resetShaderTransforms()
{
_transformArray[0] = Matrix4x4.identity;
_transformArray[1] = Matrix4x4.identity;
Shader.SetGlobalMatrixArray(HAND_ARRAY_GLOBAL_NAME, _transformArray);
}
protected virtual LeapTransform GetWarpedMatrix(long timestamp, bool updateTemporalCompensation = true)
{
if (_mainCamera == null || this == null)
{
return LeapTransform.Identity;
}
LeapTransform leapTransform = new LeapTransform();
// If temporal warping is turned off
if (_temporalWarpingMode == TemporalWarpingMode.Off)
{
//Calculate the Current Pose
Pose currentPose = Pose.identity;
if (_deviceOffsetMode == DeviceOffsetMode.Transform && deviceOrigin != null)
{
currentPose = deviceOrigin.ToPose();
}
else
{
transformHistory.SampleTransform(timestamp, out currentPose.position, out currentPose.rotation);
}
warpedPosition = currentPose.position;
warpedRotation = currentPose.rotation;
}
//Calculate a Temporally Warped Pose
else if (updateTemporalCompensation && transformHistory.history.IsFull)
{
#if SVR
if (_xr2TimewarpMode == TimewarpMode.Default && transformHistory.history.IsFull)
{
var imageAdjustment = _temporalWarpingMode == TemporalWarpingMode.Images ? -20000 : 0;
var sampleTimestamp = timestamp - (long)(warpingAdjustment * 1000f) - imageAdjustment;
transformHistory.SampleTransform(sampleTimestamp, out warpedPosition, out warpedRotation);
}
else if (_xr2TimewarpMode == TimewarpMode.Experimental_XR2)
{
// Get the predicted display time for the current frame in milliseconds, then get the predicted head pose
float predictedDisplayTime_ms = SxrShim.GetPredictedDisplayTime(SystemInfo.graphicsMultiThreaded);
Vector3 predictedWarpedPosition;
Quaternion predictedWarpedRotation;
SxrShim.GetPredictedHeadPose(predictedDisplayTime_ms, out predictedWarpedRotation, out predictedWarpedPosition);
warpedPosition.x = -predictedWarpedPosition.x;
warpedPosition.y = -predictedWarpedPosition.y;
warpedPosition.z = predictedWarpedPosition.z;
warpedRotation = predictedWarpedRotation;
}
#else
if (transformHistory.history.IsFull)
{
var imageAdjustment = _temporalWarpingMode == TemporalWarpingMode.Images ? -20000 : 0;
var sampleTimestamp = timestamp - (long)(warpingAdjustment * 1000f) - imageAdjustment;
transformHistory.SampleTransform(sampleTimestamp, out warpedPosition, out warpedRotation);
}
#endif
}
// Normalize the rotation Quaternion.
warpedRotation = warpedRotation.ToNormalized();
// If we are NOT using a transform to offset the tracking
if (_deviceOffsetMode != DeviceOffsetMode.Transform)
{
warpedPosition += warpedRotation * Vector3.up * deviceOffsetYAxis
+ warpedRotation * Vector3.forward * deviceOffsetZAxis;
warpedRotation *= Quaternion.Euler(deviceTiltXAxis, 0f, 0f);
warpedRotation *= Quaternion.Euler(-90f, 180f, 0f);
}
else
{
warpedRotation *= Quaternion.Euler(-90f, 90f, 90f);
}
#if !SVR
// Use the _mainCamera parent to transfrom the warped positions so the player can move around
if (_mainCamera.transform.parent != null)
{
leapTransform = new LeapTransform(
_mainCamera.transform.parent.TransformPoint(warpedPosition).ToVector(),
_mainCamera.transform.parent.TransformRotation(warpedRotation).ToLeapQuaternion(),
Vector.Ones * 1e-3f
);
}
else
#endif
{
leapTransform = new LeapTransform(
warpedPosition.ToVector(),
warpedRotation.ToLeapQuaternion(),
Vector.Ones * 1e-3f
);
}
leapTransform.MirrorZ();
return leapTransform;
}
protected void transformHands(ref LeapTransform LeftHand, ref LeapTransform RightHand)
{
LeapTransform leapTransform = GetWarpedMatrix(0, false);
LeftHand = new LeapTransform(leapTransform.TransformPoint(LeftHand.translation),
leapTransform.TransformQuaternion(LeftHand.rotation));
RightHand = new LeapTransform(leapTransform.TransformPoint(RightHand.translation),
leapTransform.TransformQuaternion(RightHand.rotation));
}
protected void OnPreCullHandTransforms(Camera camera)
{
if (updateHandInPrecull)
{
//Don't update pre cull for preview, reflection, or scene view cameras
if (camera == null)
{
camera = _mainCamera;
}
switch (camera.cameraType)
{
case CameraType.Preview:
#if UNITY_2017_1_OR_NEWER
case CameraType.Reflection:
#endif
case CameraType.SceneView:
return;
}
if (Application.isPlaying && !manualUpdateHasBeenCalledSinceUpdate && _leapController != null)
{
manualUpdateHasBeenCalledSinceUpdate = true;
//Find the left and/or right hand(s) to latch
Hand leftHand = null, rightHand = null;
LeapTransform precullLeftHand = LeapTransform.Identity;
LeapTransform precullRightHand = LeapTransform.Identity;
for (int i = 0; i < CurrentFrame.Hands.Count; i++)
{
Hand updateHand = CurrentFrame.Hands[i];
if (updateHand.IsLeft && leftHand == null)
{
leftHand = updateHand;
}
else if (updateHand.IsRight && rightHand == null)
{
rightHand = updateHand;
}
}
//Determine their new Transforms
var interpolationTime = CalculateInterpolationTime();
_leapController.GetInterpolatedLeftRightTransform(
interpolationTime + (ExtrapolationAmount * 1000),
interpolationTime - (BounceAmount * 1000),
(leftHand != null ? leftHand.Id : 0),
(rightHand != null ? rightHand.Id : 0),
out precullLeftHand,
out precullRightHand);
bool leftValid = precullLeftHand.translation != Vector.Zero;
bool rightValid = precullRightHand.translation != Vector.Zero;
transformHands(ref precullLeftHand, ref precullRightHand);
//Calculate the delta Transforms
if (rightHand != null && rightValid)
{
_transformArray[0] =
Matrix4x4.TRS(precullRightHand.translation.ToVector3(),
precullRightHand.rotation.ToQuaternion(),
Vector3.one)
* Matrix4x4.Inverse(Matrix4x4.TRS(rightHand.PalmPosition.ToVector3(),
rightHand.Rotation.ToQuaternion(),
Vector3.one));
}
if (leftHand != null && leftValid)
{
_transformArray[1] =
Matrix4x4.TRS(precullLeftHand.translation.ToVector3(),
precullLeftHand.rotation.ToQuaternion(),
Vector3.one)
* Matrix4x4.Inverse(Matrix4x4.TRS(leftHand.PalmPosition.ToVector3(),
leftHand.Rotation.ToQuaternion(),
Vector3.one));
}
//Apply inside of the vertex shader
Shader.SetGlobalMatrixArray(HAND_ARRAY_GLOBAL_NAME, _transformArray);
}
}
}
#if SVR
///
/// Return the predicted display time as a leap time
///
///
private long GetPredictedDisplayTime_LeapTime()
{
long leapClock = 0;
// Predicted display time for the current frame in milliseconds
float displayTime_ms = SxrShim.GetPredictedDisplayTime(SystemInfo.graphicsMultiThreaded);
if (_clockRebaser != IntPtr.Zero)
{
LeapC.RebaseClock(_clockRebaser, (long)displayTime_ms + _stopwatch.ElapsedMilliseconds, out leapClock);
}
return leapClock;
}
#endif
#endregion
}
}