using MathNet.Numerics.LinearAlgebra; using MathNet.Numerics.LinearAlgebra.Double; using UnityEngine; namespace SMPLModel { /// /// This Class takes a set of an individual body's shape-betas and /// calculates the correct skeleton for that body. /// /// It depends on a joint template combined with a /// "Regressor" matrix, which is a slightly converted version of J_template in the MPI model's documentation. /// Instead of calculating the joint locations from the vertices, as described by MPI in their python code: /// J_locations = J_Regressor * shaped_verticies /// /// Here, we can't read vertices directly because Unity does not keep consistent vertex number nor vertex index. /// So we pre-computed J_template from the verticies using a python script, /// which is saved then read from a JSON file. So we use the formula: /// /// J_locations = J_template + JRegressor * Betas /// /// where J_template is dimensions [NumJoints, 3] (Avg mesh joint locations) /// JRegressor is dimensions [NumJoints, 3, NumBetas] /// betas is dimensions [NumBetas,1] /// public class JointRegressor { readonly Matrix jointTemplate; readonly Matrix jointRegressorMatrixX; readonly Matrix jointRegressorMatrixY; readonly Matrix jointRegressorMatrixZ; public JointRegressor(Matrix jointTemplate, Matrix jointRegressorMatrixX, Matrix jointRegressorMatrixY, Matrix jointRegressorMatrixZ) { this.jointTemplate = jointTemplate; this.jointRegressorMatrixX = jointRegressorMatrixX; this.jointRegressorMatrixY = jointRegressorMatrixY; this.jointRegressorMatrixZ = jointRegressorMatrixZ; } public Vector3[] JointPositionFrom(ModelDefinition model, float[] betasArray) { Matrix betas = ConvertBetasToMatrix(betasArray); Matrix finalNewPositionMatrix = jointTemplate + J_RegressorDotBetas(betas); Vector3[] positionsInMayaCoords = MatrixToPositionArray(model, finalNewPositionMatrix); Vector3[] positions = ConvertToUnityCoordinateSystem(positionsInMayaCoords); return positions; } Vector3[] ConvertToUnityCoordinateSystem(Vector3[] jointPositions) { Vector3[] flippedJointPositions = new Vector3[jointPositions.Length]; for (int index = 0; index < jointPositions.Length; index++) { Vector3 jointPosition = jointPositions[index]; flippedJointPositions[index] = new Vector3(-jointPosition.x, jointPosition.y, jointPosition.z); } return flippedJointPositions; } /// /// Since C# lacks "einsum" function, have to do complex matrix multiplication manually /// /// this replaces python the formula: /// result = np.einsum('ijk,kl->ij', J_regressor, betas) /// /// Matrix J_RegressorDotBetas(Matrix betas) { Matrix dimensionResultsX = jointRegressorMatrixX.Multiply(betas); Matrix dimensionResultsY = jointRegressorMatrixY.Multiply(betas); Matrix dimensionResultsZ = jointRegressorMatrixZ.Multiply(betas); Matrix multiplicationResult = StackDimensionsBackToOneMatrix( dimensionResultsX, dimensionResultsY, dimensionResultsZ); return multiplicationResult; } /// /// Since C# lacks good matrix manipulation, have to stack manually. /// Basically just converts 3 separate XYZ matricies (from multiplication) /// back in to a 3-dimensional matrix /// static Matrix StackDimensionsBackToOneMatrix(Matrix dimensionResultsX, Matrix dimensionResultsY, Matrix dimensionResultsZ ) { Vector[] columnsToStack = { dimensionResultsX.Column(0), dimensionResultsY.Column(0), dimensionResultsZ.Column(0) }; Matrix stackedColumns = DenseMatrix.OfColumns(columnsToStack); return stackedColumns; } /// /// Converts array of betas to /// static Matrix ConvertBetasToMatrix(float[] betaArray) { double[,] betaMatrix = new double[betaArray.Length, 1]; for (int i = 0; i < betaArray.Length; i++) { betaMatrix[i, 0] = betaArray[i]; } Matrix betas = DenseMatrix.OfArray(betaMatrix); return betas; } /// /// Converts Matrix Back to array of Vector3 positions /// static Vector3[] MatrixToPositionArray(ModelDefinition model, Matrix positionMatrix) { Vector3[] positions = new Vector3[model.JointCount]; for (int jointIndex = 0; jointIndex < model.JointCount; jointIndex++) { Vector3 newPosition = new Vector3 { x = (float) positionMatrix[jointIndex, 0], y = (float) positionMatrix[jointIndex, 1], z = (float) positionMatrix[jointIndex, 2] }; positions[jointIndex] = newPosition; } return positions; } } }