/****************************************************************************** * Copyright (C) Ultraleap, Inc. 2011-2021. * * * * Use subject to the terms of the Apache License 2.0 available at * * http://www.apache.org/licenses/LICENSE-2.0, or another agreement * * between Ultraleap and you, your company or other organization. * ******************************************************************************/ using Leap.Unity.Splines; using System; using UnityEngine; namespace Leap.Unity.Encoding { /// /// An interface that signifies this class can interpolate /// via the standard techniques /// public interface IInterpolable { T CopyFrom(T toCopy); bool FillLerped(T from, T to, float t); bool FillSplined(T a, T b, T c, T d, float t); } /// /// A Vector-based encoding of a Leap Hand. /// /// You can Encode a VectorHand from a Leap hand, Decode a VectorHand into a Leap hand, /// convert the VectorHand to a compressed byte representation using FillBytes, /// and decompress back into a VectorHand using FromBytes. /// /// Also see CurlHand for a more compressed but slightly less articulated encoding. /// TODO: CurlHand not yet brought in from Networking module! /// [Serializable] public class VectorHand : IInterpolable { #region Data public const int NUM_JOINT_POSITIONS = 25; public bool isLeft; public Vector3 palmPos; public Quaternion palmRot; public Pose palmPose { get { return new Pose(palmPos, palmRot); } } [SerializeField] private Vector3[] _backingJointPositions; public Vector3[] jointPositions { get { if (_backingJointPositions == null || _backingJointPositions.Length != NUM_JOINT_POSITIONS) { _backingJointPositions = new Vector3[NUM_JOINT_POSITIONS]; } return _backingJointPositions; } } #endregion public VectorHand() { } /// /// Constructs a VectorHand representation from a Leap hand. This allocates a vector /// array for the encoded hand data. /// /// Use a pooling strategy to avoid unnecessary allocation in runtime contexts. /// public VectorHand(Hand hand) : this() { Encode(hand); } /// /// Copies a VectorHand from another VectorHand /// public VectorHand CopyFrom(VectorHand h) { if (h != null) { isLeft = h.isLeft; palmPos = h.palmPos; palmRot = h.palmRot; for (int i = 0; i < jointPositions.Length; i++) _backingJointPositions[i] = h.jointPositions[i]; } return this; } #region Hand Encoding public void Encode(Hand fromHand) { isLeft = fromHand.IsLeft; palmPos = fromHand.PalmPosition.ToVector3(); palmRot = fromHand.Rotation.ToQuaternion(); int boneIdx = 0; for (int i = 0; i < 5; i++) { Vector3 baseMetacarpal = ToLocal( fromHand.Fingers[i].bones[0].PrevJoint.ToVector3(), palmPos, palmRot); jointPositions[boneIdx++] = baseMetacarpal; for (int j = 0; j < 4; j++) { Vector3 joint = ToLocal( fromHand.Fingers[i].bones[j].NextJoint.ToVector3(), palmPos, palmRot); jointPositions[boneIdx++] = joint; } } } // TODO: DELETEME public static Vector3 tweakWristPosition = new Vector3(0f, -0.015f, -0.065f); public void Decode(Hand intoHand) { int boneIdx = 0; Vector3 prevJoint = Vector3.zero; Vector3 nextJoint = Vector3.zero; Quaternion boneRot = Quaternion.identity; // Fill fingers. for (int fingerIdx = 0; fingerIdx < 5; fingerIdx++) { for (int jointIdx = 0; jointIdx < 4; jointIdx++) { boneIdx = fingerIdx * 4 + jointIdx; prevJoint = jointPositions[fingerIdx * 5 + jointIdx]; nextJoint = jointPositions[fingerIdx * 5 + jointIdx + 1]; if ((nextJoint - prevJoint).normalized == Vector3.zero) { // Thumb "metacarpal" slot is an identity bone. boneRot = Quaternion.identity; } else { boneRot = Quaternion.LookRotation( (nextJoint - prevJoint).normalized, Vector3.Cross((nextJoint - prevJoint).normalized, (fingerIdx == 0 ? (isLeft ? -Vector3.up : Vector3.up) : Vector3.right))); } // Convert to world space from palm space. nextJoint = ToWorld(nextJoint, palmPos, palmRot); prevJoint = ToWorld(prevJoint, palmPos, palmRot); boneRot = palmRot * boneRot; intoHand.GetBone(boneIdx).Fill( prevJoint: prevJoint.ToVector(), nextJoint: nextJoint.ToVector(), center: ((nextJoint + prevJoint) / 2f).ToVector(), direction: (palmRot * Vector3.forward).ToVector(), length: (prevJoint - nextJoint).magnitude, width: 0.01f, type: (Bone.BoneType)jointIdx, rotation: boneRot.ToLeapQuaternion()); } intoHand.Fingers[fingerIdx].Fill( frameId: -1, handId: (isLeft ? 0 : 1), fingerId: fingerIdx, timeVisible: 10f,// Time.time, <- This is unused and main thread only tipPosition: nextJoint.ToVector(), direction: (boneRot * Vector3.forward).ToVector(), width: 1f, length: 1f, isExtended: true, type: (Finger.FingerType)fingerIdx); } // Fill arm data. intoHand.Arm.Fill(ToWorld(new Vector3(0f, 0f, -0.3f), palmPos, palmRot).ToVector(), ToWorld(new Vector3(0f, 0f, -0.055f), palmPos, palmRot).ToVector(), ToWorld(new Vector3(0f, 0f, -0.125f), palmPos, palmRot).ToVector(), Vector.Zero, 0.3f, 0.05f, (palmRot).ToLeapQuaternion()); // Finally, fill hand data. var palmPose = new Pose(palmPos, palmRot); // var wristPos = ToWorld(new Vector3(0f, -0.015f, -0.065f), palmPos, palmRot); var wristPos = palmPose.mul(tweakWristPosition).position; intoHand.Fill( frameID: -1, id: (isLeft ? 0 : 1), confidence: 1f, grabStrength: 0.5f, grabAngle: 100f, pinchStrength: 0.5f, pinchDistance: 50f, palmWidth: 0.085f, isLeft: isLeft, timeVisible: 1f, fingers: null /* already uploaded finger data */, palmPosition: palmPos.ToVector(), stabilizedPalmPosition: palmPos.ToVector(), palmVelocity: Vector3.zero.ToVector(), palmNormal: (palmRot * Vector3.down).ToVector(), rotation: (palmRot.ToLeapQuaternion()), direction: (palmRot * Vector3.forward).ToVector(), wristPosition: wristPos.ToVector() ); // TODO: DELETEME // var sphere = new Geometry.Sphere(radius: 0.008f); // var drawer = HyperMegaStuff.HyperMegaLines.drawer; // drawer.color = LeapColor.cerulean; // sphere.WithCenter(wristPos).DrawLines(drawer.DrawLine); // sphere.radius = 0.007f; // drawer.color = LeapColor.white; // foreach (var point in jointPositions) { // sphere.WithCenter((palmPose * point).position).DrawLines(drawer.DrawLine); // } } #endregion #region Byte Encoding & Decoding /// /// The number of bytes required to encode a VectorHand into its byte representation. /// The byte representation is compressed to 86 bytes. /// /// The first byte determines chirality, the camera-local hand position uses 6 bytes, /// the camera-local hand rotation uses 4 bytes, and each joint position component is /// encoded in hand-local space using 3 bytes. /// public int numBytesRequired { get { return 86; } } public const int NUM_BYTES = 86; /// /// Fills this VectorHand with data read from the provided byte array, starting at /// the provided offset. /// public void ReadBytes(byte[] bytes, int offset = 0) { ReadBytes(bytes, ref offset); } /// /// Fills this VectorHand with data read from the provided byte array, starting at /// the provided offset. /// public void ReadBytes(byte[] bytes, ref int offset) { if (bytes.Length - offset < numBytesRequired) { throw new System.IndexOutOfRangeException( "Not enough room to read bytes for VectorHand encoding starting at offset " + offset + " for array of size " + bytes + "; need at least " + numBytesRequired + " bytes from the offset position."); } // Chirality. isLeft = bytes[offset++] == 0x00; // Palm position and rotation. for (int i = 0; i < 3; i++) { palmPos[i] = Convert.ToSingle( BitConverterNonAlloc.ToInt16(bytes, ref offset)) / 4096f; } palmRot = Utils.DecompressBytesToQuat(bytes, ref offset); // Palm-local bone joint positions. for (int i = 0; i < NUM_JOINT_POSITIONS; i++) { for (int j = 0; j < 3; j++) { jointPositions[i][j] = VectorHandExtensions.ByteToFloat(bytes[offset++]); } } } /// /// Fills the provided byte array with a compressed, 86-byte form of this VectorHand, /// starting at the provided offset. /// /// Throws an IndexOutOfRangeException if the provided byte array doesn't have enough /// space (starting from the offset) to write the number of bytes required. /// public void FillBytes(byte[] bytesToFill, ref int offset) { if (_backingJointPositions == null) { throw new System.InvalidOperationException( "Joint positions array is null. You must fill a VectorHand with data before " + "you can use it to fill byte representations."); } if (bytesToFill.Length - offset < numBytesRequired) { throw new System.IndexOutOfRangeException( "Not enough room to fill bytes for VectorHand encoding starting at offset " + offset + " for array of size " + bytesToFill.Length + "; need at least " + numBytesRequired + " bytes from the offset position."); } // Chirality. bytesToFill[offset++] = isLeft ? (byte)0x00 : (byte)0x01; // Palm position, each component compressed for (int i = 0; i < 3; i++) { BitConverterNonAlloc.GetBytes(Convert.ToInt16(palmPos[i] * 4096f), bytesToFill, ref offset); } // Palm rotation. Utils.CompressQuatToBytes(palmRot, bytesToFill, ref offset); // Joint positions. for (int j = 0; j < NUM_JOINT_POSITIONS; j++) { for (int i = 0; i < 3; i++) { bytesToFill[offset++] = VectorHandExtensions.FloatToByte(jointPositions[j][i]); } } } /// /// Fills the provided byte array with a compressed, 86-byte form of this VectorHand. /// /// Throws an IndexOutOfRangeException if the provided byte array doesn't have enough /// space to write the number of bytes required (see VectorHand.BYTE_ENCODING_SIZE). /// public void FillBytes(byte[] bytesToFill) { int unusedOffset = 0; FillBytes(bytesToFill, ref unusedOffset); } /// /// Shortcut for reading a VectorHand-encoded byte representation of a Leap hand and /// decoding it immediately into a Hand object. /// public void ReadBytes(byte[] bytes, ref int offset, Hand intoHand) { ReadBytes(bytes, ref offset); Decode(intoHand); } /// /// Shortcut for encoding a Leap hand into a VectorHand representation and /// compressing it immediately into a byte representation. /// If the provided Hand is null, the 86 bytes are set to zero. /// public void FillBytes(byte[] bytes, ref int offset, Hand fromHand) { if (fromHand == null) { for (int i = offset; i < offset + NUM_BYTES; i++) { bytes[i] = 0; } } else { Encode(fromHand); FillBytes(bytes, ref offset); } } [ThreadStatic] private static VectorHand s_backingCachedVectorHand; private static VectorHand s_cachedVectorHand { get { if (s_backingCachedVectorHand == null) { s_backingCachedVectorHand = new VectorHand(); } return s_backingCachedVectorHand; } } [ThreadStatic] private static Vector3[] s_backingJointsBuffer = new Vector3[NUM_JOINT_POSITIONS]; private static Vector3[] s_jointsBuffer { get { if (s_backingJointsBuffer == null) { s_backingJointsBuffer = new Vector3[NUM_JOINT_POSITIONS]; } return s_backingJointsBuffer; } } /// /// Fills bytes using a thread-safe (ThreadStatic) cached VectorHand to /// encode the provided Hand. /// If the provided Hand is null, the 86 bytes are set to zero. /// public static void StaticFillBytes(byte[] bytes, Hand fromHand) { StaticFillBytes(bytes, 0, fromHand); } /// /// Fills bytes at the argument offset using a thread-safe (ThreadStatic) /// cached VectorHand to encode the provided Hand. /// If the provided Hand is null, the 86 bytes are set to zero. /// public static void StaticFillBytes(byte[] bytes, int offset, Hand fromHand) { StaticFillBytes(bytes, ref offset, fromHand); } /// /// Fills bytes at the argument offset using a thread-safe (ThreadStatic) /// cached VectorHand to encode the provided Hand. /// If the provided Hand is null, the 86 bytes are set to zero. /// public static void StaticFillBytes(byte[] bytes, ref int offset, Hand fromHand) { s_cachedVectorHand._backingJointPositions = s_jointsBuffer; s_cachedVectorHand.FillBytes(bytes, ref offset, fromHand); } #endregion #region Utility /// /// Converts a local-space point to a world-space point given the local space's /// origin and rotation. /// public static Vector3 ToWorld(Vector3 localPoint, Vector3 localOrigin, Quaternion localRot) { return (localRot * localPoint) + localOrigin; } /// /// Converts a world-space point to a local-space point given the local /// space's origin and rotation. /// public static Vector3 ToLocal(Vector3 worldPoint, Vector3 localOrigin, Quaternion localRot) { return Quaternion.Inverse(localRot) * (worldPoint - localOrigin); } /// Fills the ref-argument VectorHand with interpolated data /// between the two other VectorHands, by t (unclamped), and return true. /// If either a or b is null, the ref-argument VectorHand is also set to /// null, and the method returns false. /// An exception is thrown if the interpolation arguments a and b don't /// have the same chirality. /// public bool FillLerped(VectorHand a, VectorHand b, float t) { if (a == null || b == null) return false; if (a.isLeft != b.isLeft) { throw new System.Exception("VectorHands must be interpolated with the " + "same chirality."); } isLeft = a.isLeft; palmPos = Vector3.LerpUnclamped(a.palmPos, b.palmPos, t); palmRot = Quaternion.SlerpUnclamped(a.palmRot, b.palmRot, t); for (int i = 0; i < jointPositions.Length; i++) { jointPositions[i] = Vector3.LerpUnclamped(a.jointPositions[i], b.jointPositions[i], t); } return true; } /// Fills the ref-argument VectorHand with interpolated data /// between the 4 other VectorHands, by t (unclamped), and return true. /// If either a, b, c or d is null, the ref-argument VectorHand is also set to /// null, and the method returns false. /// An exception is thrown if the interpolation arguments a and b don't /// have the same chirality. /// public bool FillSplined(VectorHand a, VectorHand b, VectorHand c, VectorHand d, float t) { if (a == null || b == null || c == null || d == null) { return false; } if (b.isLeft != c.isLeft) { throw new System.Exception("VectorHands must be interpolated with the " + "same chirality."); } isLeft = a.isLeft; palmPos = CatmullRom.ToCHS( a.palmPos, b.palmPos, c.palmPos, d.palmPos, false).PositionAt(t); palmRot = Quaternion.SlerpUnclamped(b.palmRot, c.palmRot, t); //Quaternion splines are not as robust //CatmullRom.ToQuaternionCHS( // a.palmRot, b.palmRot, c.palmRot, d.palmRot, false).RotationAt(t); for (int i = 0; i < jointPositions.Length; i++) { jointPositions[i] = CatmullRom.ToCHS( a.jointPositions[i], b.jointPositions[i], c.jointPositions[i], d.jointPositions[i], false).PositionAt(t); } return true; } #endregion } #region Utility Extension Methods public static class VectorHandExtensions { #region VectorHand Instance API //public static void FillBytes(this VectorHand vectorHand, ) #endregion #region Utilities /// /// Returns a bone object from the hand as if all bones were aligned metacarpal- /// to-tip and thumb-to-pinky. So 0-3 represent thumb bones, 4-7 represent index /// bones, etc. There are 20 such Bones in a Hand. /// public static Bone GetBone(this Hand hand, int boneIdx) { return hand.Fingers[boneIdx / 4].bones[boneIdx % 4]; } /// /// Compresses a float into a byte based on the desired movement range. /// public static byte FloatToByte(float inFloat, float movementRange = 0.3f) { float clamped = Mathf.Clamp(inFloat, -movementRange / 2f, movementRange / 2f); clamped += movementRange / 2f; clamped /= movementRange; clamped *= 255f; clamped = Mathf.Floor(clamped); return (byte)clamped; } /// /// Expands a byte back into a float based on the desired movement range. /// public static float ByteToFloat(byte inByte, float movementRange = 0.3f) { float clamped = (float)inByte; clamped /= 255f; clamped *= movementRange; clamped -= movementRange / 2f; return clamped; } #endregion } #endregion }