/******************************************************************************
* Copyright (C) Ultraleap, Inc. 2011-2021. *
* *
* Use subject to the terms of the Apache License 2.0 available at *
* http://www.apache.org/licenses/LICENSE-2.0, or another agreement *
* between Ultraleap and you, your company or other organization. *
******************************************************************************/
using Leap.Unity.Query;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.SceneManagement;
namespace Leap.Unity
{
///
/// Static convenience methods and extension methods for getting useful Hand data.
///
public static class Hands
{
private static LeapProvider s_provider;
private static GameObject s_leapRig;
static Hands()
{
InitStatic();
SceneManager.activeSceneChanged += InitStaticOnNewScene;
}
private static void InitStaticOnNewScene(Scene unused, Scene unused2)
{
InitStatic();
}
private static void InitStatic()
{
s_provider = Object.FindObjectOfType();
if (s_provider == null)
{
s_provider = Object.FindObjectOfType();
if (s_provider == null)
{
return;
}
}
Camera providerCamera = s_provider.GetComponentInParent();
if (providerCamera == null) return;
if (providerCamera.transform.parent == null) return;
s_leapRig = providerCamera.transform.parent.gameObject;
}
///
/// Static convenience accessor for the Leap camera rig. This is the parent
/// of the Camera that contains a LeapProvider in one of its children,
/// or null if there is no such GameObject.
///
public static GameObject CameraRig
{
get
{
if (s_leapRig == null)
{
InitStatic();
}
return s_leapRig;
}
}
///
/// Static convenience accessor for a LeapProvider in the scene. Preference is given
/// to a LeapServiceProvider if there is one.
///
/// If static memory currently has no reference for the provider (or if it was
/// destroyed), this call will search the scene for a LeapProvider and cache it to be
/// returned next time.
///
/// If there is no LeapProvider in your scene, this getter
/// will return null. Be warned that calling this regularly can be expensive if
/// LeapProviders often don't exist in your scene or are frequently destroyed.
///
public static LeapProvider Provider
{
get
{
if (s_provider == null)
{
InitStatic();
}
return s_provider;
}
set { s_provider = value; }
}
///
/// Returns the first hand of the argument Chirality in the current frame,
/// otherwise returns null if no such hand is found.
///
public static Hand Get(Chirality chirality)
{
if (chirality == Chirality.Left) return Left;
else return Right;
}
///
/// As Get, but returns the FixedUpdate (physics timestep) hand as opposed to the Update hand.
///
public static Hand GetFixed(Chirality chirality)
{
if (chirality == Chirality.Left) return FixedLeft;
else return FixedRight;
}
///
/// Returns the first left hand found by Leap in the current frame, otherwise
/// returns null if no such hand is found.
///
public static Hand Left
{
get
{
if (Provider == null) return null;
if (Provider.CurrentFrame == null) return null;
return Provider.CurrentFrame.Hands.Query().FirstOrDefault(hand => hand.IsLeft);
}
}
///
/// Returns the first right hand found by Leap in the current frame, otherwise
/// returns null if no such hand is found.
///
public static Hand Right
{
get
{
if (Provider == null) return null;
if (Provider.CurrentFrame == null) return null;
else return Provider.CurrentFrame.Hands.Query().FirstOrDefault(hand => hand.IsRight);
}
}
///
/// Returns the first left hand found by Leap in the current fixed frame, otherwise
/// returns null if no such hand is found. The fixed frame is aligned with the physics timestep.
///
public static Hand FixedLeft
{
get
{
if (Provider == null) return null;
if (Provider.CurrentFixedFrame == null) return null;
return Provider.CurrentFixedFrame.Hands.Query().FirstOrDefault(hand => hand.IsLeft);
}
}
///
/// Returns the first right hand found by Leap in the current fixed frame, otherwise
/// returns null if no such hand is found. The fixed frame is aligned with the physics timestep.
///
public static Hand FixedRight
{
get
{
if (Provider == null) return null;
if (Provider.CurrentFixedFrame == null) return null;
else return Provider.CurrentFixedFrame.Hands.Query().FirstOrDefault(hand => hand.IsRight);
}
}
/// Shorthand for hand.Fingers[(int)Leap.Finger.FingerType.TYPE_THUMB],
/// or, alternatively, hand.Fingers[0].
///
public static Finger GetThumb(this Hand hand)
{
return hand.Fingers[(int)Leap.Finger.FingerType.TYPE_THUMB];
}
///
/// Shorthand for hand.Fingers[(int)Leap.Finger.FingerType.TYPE_INDEX],
/// or, alternatively, hand.Fingers[1].
///
public static Finger GetIndex(this Hand hand)
{
return hand.Fingers[(int)Leap.Finger.FingerType.TYPE_INDEX];
}
///
/// Shorthand for hand.Fingers[(int)Leap.Finger.FingerType.TYPE_MIDDLE],
/// or, alternatively, hand.Fingers[2].
///
public static Finger GetMiddle(this Hand hand)
{
return hand.Fingers[(int)Leap.Finger.FingerType.TYPE_MIDDLE];
}
///
/// Shorthand for hand.Fingers[(int)Leap.Finger.FingerType.TYPE_RING],
/// or, alternatively, hand.Fingers[3].
///
public static Finger GetRing(this Hand hand)
{
return hand.Fingers[(int)Leap.Finger.FingerType.TYPE_RING];
}
///
/// Shorthand for hand.Fingers[(int)Leap.Finger.FingerType.TYPE_PINKY],
/// or, alternatively, hand.Fingers[4].
///
public static Finger GetPinky(this Hand hand)
{
return hand.Fingers[(int)Leap.Finger.FingerType.TYPE_PINKY];
}
///
/// Returns a Pose consisting of the tracked hand's palm position and rotation.
///
public static Pose GetPalmPose(this Hand hand)
{
return new Pose(hand.PalmPosition.ToVector3(), hand.Rotation.ToQuaternion());
}
///
/// As Hand.SetTransform(), but takes a Pose as input for convenience.
///
public static void SetPalmPose(this Hand hand, Pose newPalmPose)
{
hand.SetTransform(newPalmPose.position, newPalmPose.rotation);
}
///
/// Returns the direction the Hand's palm is facing. For the other two palm-basis
/// directions, see RadialAxis and DistalAxis.
///
/// The direction out of the back of the hand would be called the dorsal axis.
///
public static Vector3 PalmarAxis(this Hand hand)
{
return -hand.Basis.yBasis.ToVector3();
}
///
/// Returns the the direction towards the thumb that is perpendicular to the palmar
/// and distal axes. Left and right hands will return opposing directions.
///
/// The direction away from the thumb would be called the ulnar axis.
///
public static Vector3 RadialAxis(this Hand hand)
{
if (hand.IsRight)
{
return -hand.Basis.xBasis.ToVector3();
}
else
{
return hand.Basis.xBasis.ToVector3();
}
}
///
/// Returns the direction towards the fingers that is perpendicular to the palmar
/// and radial axes.
///
/// The direction towards the wrist would be called the proximal axis.
///
public static Vector3 DistalAxis(this Hand hand)
{
return hand.Basis.zBasis.ToVector3();
}
///
/// Returns whether the pinch strength for the hand is greater than 0.8.
/// For more reliable pinch behavior, try applying hysteresis to the PinchStrength property.
///
public static bool IsPinching(this Hand hand)
{
return hand.PinchStrength > 0.8F;
}
///
/// Returns approximately where the thumb and index finger will be if they are pinched together.
///
public static Vector3 GetPinchPosition(this Hand hand)
{
Vector indexPosition = hand.Fingers[(int)Finger.FingerType.TYPE_INDEX].TipPosition;
Vector thumbPosition = hand.Fingers[(int)Finger.FingerType.TYPE_THUMB].TipPosition;
return (2 * thumbPosition + indexPosition).ToVector3() * 0.333333F;
}
///
/// Returns a decent approximation of where the hand is pinching, or where it will pinch,
/// even if the index and thumb tips are far apart.
///
/// In general, this will be more stable than GetPinchPosition().
///
public static Vector3 GetPredictedPinchPosition(this Hand hand)
{
Vector3 indexTip = hand.GetIndex().TipPosition.ToVector3();
Vector3 thumbTip = hand.GetThumb().TipPosition.ToVector3();
// The predicted pinch point is a rigid point in hand-space linearly offset by the
// index finger knuckle position, scaled by the index finger's length, and lightly
// influenced by the actual thumb and index tip positions.
Vector3 indexKnuckle = hand.Fingers[1].bones[1].PrevJoint.ToVector3();
float indexLength = hand.Fingers[1].Length;
Vector3 radialAxis = hand.RadialAxis();
float thumbInfluence = Vector3.Dot((thumbTip - indexKnuckle).normalized, radialAxis).Map(0F, 1F, 0.5F, 0F);
Vector3 predictedPinchPoint = indexKnuckle + hand.PalmarAxis() * indexLength * 0.85F
+ hand.DistalAxis() * indexLength * 0.20F
+ radialAxis * indexLength * 0.20F;
predictedPinchPoint = Vector3.Lerp(predictedPinchPoint, thumbTip, thumbInfluence);
predictedPinchPoint = Vector3.Lerp(predictedPinchPoint, indexTip, 0.15F);
return predictedPinchPoint;
}
///
/// Returns whether this vector faces from a given world position towards another world position within a maximum angle of error.
///
public static bool IsFacing(this Vector3 facingVector, Vector3 fromWorldPosition, Vector3 towardsWorldPosition, float maxOffsetAngleAllowed)
{
Vector3 actualVectorTowardsWorldPosition = (towardsWorldPosition - fromWorldPosition).normalized;
return Vector3.Angle(facingVector, actualVectorTowardsWorldPosition) <= maxOffsetAngleAllowed;
}
///
/// Returns a confidence value from 0 to 1 indicating how strongly the Hand is making a fist.
///
public static float GetFistStrength(this Hand hand)
{
return (Vector3.Dot(hand.Fingers[1].Direction.ToVector3(), -hand.DistalAxis())
+ Vector3.Dot(hand.Fingers[2].Direction.ToVector3(), -hand.DistalAxis())
+ Vector3.Dot(hand.Fingers[3].Direction.ToVector3(), -hand.DistalAxis())
+ Vector3.Dot(hand.Fingers[4].Direction.ToVector3(), -hand.DistalAxis())
+ Vector3.Dot(hand.Fingers[0].Direction.ToVector3(), -hand.RadialAxis())
).Map(-5, 5, 0, 1);
}
///
/// Returns an unsmoothed ray representing the general reaching/interaction intent direction.
///
public static Ray HandRay(this Hand hand, Transform headTransform)
{
Quaternion shoulderYaw = Quaternion.Euler(0f, headTransform.rotation.eulerAngles.y, 0f);
// Approximate shoulder position with magic values.
Vector3 ProjectionOrigin = headTransform.position
+ (shoulderYaw * (new Vector3(0f, -0.13f, -0.1f)
+ Vector3.left * 0.15f * (hand.IsLeft ? 1f : -1f)));
// Compare against this
//Vector3 ProjectionOrigin = headTransform.position + shoulderYaw *
// new Vector3(0.15f * (hand.IsLeft ? -1f : 1f), -0.13f, 0.05f);
Vector3 ProjectionDirection = hand.Fingers[1].bones[0].NextJoint.ToVector3() - ProjectionOrigin;
return new Ray(ProjectionOrigin, ProjectionDirection);
}
///
/// Transforms a bone by a position and rotation.
///
public static void Transform(this Bone bone, Vector3 position, Quaternion rotation)
{
bone.Transform(new LeapTransform(position.ToVector(), rotation.ToLeapQuaternion()));
}
///
/// Transforms a finger by a position and rotation.
///
public static void Transform(this Finger finger, Vector3 position, Quaternion rotation)
{
finger.Transform(new LeapTransform(position.ToVector(), rotation.ToLeapQuaternion()));
}
///
/// Transforms a hand by a position and rotation.
///
public static void Transform(this Hand hand, Vector3 position, Quaternion rotation)
{
hand.Transform(new LeapTransform(position.ToVector(), rotation.ToLeapQuaternion()));
}
///
/// Transforms a frame by a position and rotation.
///
public static void Transform(this Frame frame, Vector3 position, Quaternion rotation)
{
frame.Transform(new LeapTransform(position.ToVector(), rotation.ToLeapQuaternion()));
}
///
/// Transforms a bone to a position and rotation.
///
public static void SetTransform(this Bone bone, Vector3 position, Quaternion rotation)
{
bone.Transform(Vector3.zero, (rotation * Quaternion.Inverse(bone.Rotation.ToQuaternion())));
bone.Transform(position - bone.PrevJoint.ToVector3(), Quaternion.identity);
}
///
/// Transforms a finger to a position and rotation by its fingertip.
///
public static void SetTipTransform(this Finger finger, Vector3 position, Quaternion rotation)
{
finger.Transform(Vector3.zero, (rotation * Quaternion.Inverse(finger.bones[3].Rotation.ToQuaternion())));
finger.Transform(position - finger.bones[3].NextJoint.ToVector3(), Quaternion.identity);
}
///
/// Transforms a hand to a position and rotation.
///
public static void SetTransform(this Hand hand, Vector3 position, Quaternion rotation)
{
hand.Transform(Vector3.zero, Quaternion.Slerp((rotation * Quaternion.Inverse(hand.Rotation.ToQuaternion())), Quaternion.identity, 0f));
hand.Transform(position - hand.PalmPosition.ToVector3(), Quaternion.identity);
}
}
///
/// Utility methods for constructing and manipulating Leap hand object data.
///
public static class HandUtils
{
///
/// Fills the Hand object with the provided hand data. You can pass null for the
/// fingers input; this will leave the hand's finger data unmodified.
///
public static void Fill(this Hand toFill,
long frameID,
int id,
float confidence,
float grabStrength,
float grabAngle,
float pinchStrength,
float pinchDistance,
float palmWidth,
bool isLeft,
float timeVisible,
/* Arm arm,*/
List fingers,
Vector palmPosition,
Vector stabilizedPalmPosition,
Vector palmVelocity,
Vector palmNormal,
LeapQuaternion rotation,
Vector direction,
Vector wristPosition)
{
toFill.FrameId = frameID;
toFill.Id = id;
toFill.Confidence = confidence;
toFill.GrabStrength = grabStrength;
toFill.GrabAngle = grabAngle;
toFill.PinchStrength = pinchStrength;
toFill.PinchDistance = pinchDistance;
toFill.PalmWidth = palmWidth;
toFill.IsLeft = isLeft;
toFill.TimeVisible = timeVisible;
if (fingers != null) toFill.Fingers = fingers;
toFill.PalmPosition = palmPosition;
toFill.StabilizedPalmPosition = stabilizedPalmPosition;
toFill.PalmVelocity = palmVelocity;
toFill.PalmNormal = palmNormal;
toFill.Rotation = rotation;
toFill.Direction = direction;
toFill.WristPosition = wristPosition;
}
///
/// Fills the Bone object with the provided bone data.
///
public static void Fill(this Bone toFill,
Vector prevJoint,
Vector nextJoint,
Vector center,
Vector direction,
float length,
float width,
Bone.BoneType type,
LeapQuaternion rotation)
{
toFill.PrevJoint = prevJoint;
toFill.NextJoint = nextJoint;
toFill.Center = center;
toFill.Direction = direction;
toFill.Length = length;
toFill.Width = width;
toFill.Type = type;
toFill.Rotation = rotation;
}
///
/// Fills the Finger object with the provided finger data. You can pass null for
/// bones; A null bone will not modify the underlying hand's data for that bone.
///
public static void Fill(this Finger toFill,
long frameId,
int handId,
int fingerId,
float timeVisible,
Vector tipPosition,
Vector direction,
float width,
float length,
bool isExtended,
Finger.FingerType type,
Bone metacarpal = null,
Bone proximal = null,
Bone intermediate = null,
Bone distal = null)
{
toFill.Id = handId;
toFill.HandId = handId;
toFill.TimeVisible = timeVisible;
toFill.TipPosition = tipPosition;
toFill.Direction = direction;
toFill.Width = width;
toFill.Length = length;
toFill.IsExtended = isExtended;
toFill.Type = type;
if (metacarpal != null) toFill.bones[0] = metacarpal;
if (proximal != null) toFill.bones[1] = proximal;
if (intermediate != null) toFill.bones[2] = intermediate;
if (distal != null) toFill.bones[3] = distal;
}
///
/// Fills the Arm object with the provided arm data.
///
public static void Fill(this Arm toFill,
Vector elbow,
Vector wrist,
Vector center,
Vector direction,
float length,
float width,
LeapQuaternion rotation)
{
toFill.PrevJoint = elbow;
toFill.NextJoint = wrist;
toFill.Center = center;
toFill.Direction = direction;
toFill.Length = length;
toFill.Width = width;
toFill.Rotation = rotation;
}
///
/// Fills the hand's PalmVelocity data based on the
/// previous hand object and the provided delta time between the two hands.
///
public static void FillTemporalData(this Hand toFill,
Hand previousHand, float deltaTime)
{
toFill.PalmVelocity = (toFill.PalmPosition - previousHand.PalmPosition)
/ deltaTime;
}
#region Frame Utils
[System.Obsolete("Method is obsolete, use GetHand instead", true)]
public static Hand Get(this Frame frame, Chirality whichHand)
{
return GetHand(frame, whichHand);
}
public static Hand GetHand(this Frame frame, Chirality whichHand)
{
if (frame.Hands == null) { return null; }
return frame.Hands.Query().FirstOrDefault(
h => h.IsLeft == (whichHand == Chirality.Left));
}
#endregion
#region Provider Utils
public static Hand Get(this LeapProvider provider, Chirality whichHand)
{
Frame frame;
if (Time.inFixedTimeStep)
{
frame = provider.CurrentFixedFrame;
}
else
{
frame = provider.CurrentFrame;
}
return frame.GetHand(whichHand);
}
#endregion
}
}