using MathNet.Numerics.LinearAlgebra;
using MathNet.Numerics.LinearAlgebra.Double;
using UnityEngine;
namespace SMPLModel {
///
/// This Class takes a set of an individual body's shape-betas and
/// calculates the correct skeleton for that body.
///
/// It depends on a joint template combined with a
/// "Regressor" matrix, which is a slightly converted version of J_template in the MPI model's documentation.
/// Instead of calculating the joint locations from the vertices, as described by MPI in their python code:
/// J_locations = J_Regressor * shaped_verticies
///
/// Here, we can't read vertices directly because Unity does not keep consistent vertex number nor vertex index.
/// So we pre-computed J_template from the verticies using a python script,
/// which is saved then read from a JSON file. So we use the formula:
///
/// J_locations = J_template + JRegressor * Betas
///
/// where J_template is dimensions [NumJoints, 3] (Avg mesh joint locations)
/// JRegressor is dimensions [NumJoints, 3, NumBetas]
/// betas is dimensions [NumBetas,1]
///
public class JointRegressor {
readonly Matrix jointTemplate;
readonly Matrix jointRegressorMatrixX;
readonly Matrix jointRegressorMatrixY;
readonly Matrix jointRegressorMatrixZ;
public JointRegressor(Matrix jointTemplate, Matrix jointRegressorMatrixX,
Matrix jointRegressorMatrixY, Matrix jointRegressorMatrixZ) {
this.jointTemplate = jointTemplate;
this.jointRegressorMatrixX = jointRegressorMatrixX;
this.jointRegressorMatrixY = jointRegressorMatrixY;
this.jointRegressorMatrixZ = jointRegressorMatrixZ;
}
public Vector3[] JointPositionFrom(ModelDefinition model, float[] betasArray) {
Matrix betas = ConvertBetasToMatrix(betasArray);
Matrix finalNewPositionMatrix = jointTemplate + J_RegressorDotBetas(betas);
Vector3[] positionsInMayaCoords = MatrixToPositionArray(model, finalNewPositionMatrix);
Vector3[] positions = ConvertToUnityCoordinateSystem(positionsInMayaCoords);
return positions;
}
Vector3[] ConvertToUnityCoordinateSystem(Vector3[] jointPositions) {
Vector3[] flippedJointPositions = new Vector3[jointPositions.Length];
for (int index = 0; index < jointPositions.Length; index++) {
Vector3 jointPosition = jointPositions[index];
flippedJointPositions[index] = new Vector3(-jointPosition.x, jointPosition.y, jointPosition.z);
}
return flippedJointPositions;
}
///
/// Since C# lacks "einsum" function, have to do complex matrix multiplication manually
///
/// this replaces python the formula:
/// result = np.einsum('ijk,kl->ij', J_regressor, betas)
///
///
Matrix J_RegressorDotBetas(Matrix betas) {
Matrix dimensionResultsX = jointRegressorMatrixX.Multiply(betas);
Matrix dimensionResultsY = jointRegressorMatrixY.Multiply(betas);
Matrix dimensionResultsZ = jointRegressorMatrixZ.Multiply(betas);
Matrix multiplicationResult = StackDimensionsBackToOneMatrix(
dimensionResultsX,
dimensionResultsY,
dimensionResultsZ);
return multiplicationResult;
}
///
/// Since C# lacks good matrix manipulation, have to stack manually.
/// Basically just converts 3 separate XYZ matricies (from multiplication)
/// back in to a 3-dimensional matrix
///
static Matrix StackDimensionsBackToOneMatrix(Matrix dimensionResultsX,
Matrix dimensionResultsY,
Matrix dimensionResultsZ
) {
Vector[] columnsToStack = {
dimensionResultsX.Column(0),
dimensionResultsY.Column(0),
dimensionResultsZ.Column(0)
};
Matrix stackedColumns = DenseMatrix.OfColumns(columnsToStack);
return stackedColumns;
}
///
/// Converts array of betas to
///
static Matrix ConvertBetasToMatrix(float[] betaArray) {
double[,] betaMatrix = new double[betaArray.Length, 1];
for (int i = 0; i < betaArray.Length; i++) {
betaMatrix[i, 0] = betaArray[i];
}
Matrix betas = DenseMatrix.OfArray(betaMatrix);
return betas;
}
///
/// Converts Matrix Back to array of Vector3 positions
///
static Vector3[] MatrixToPositionArray(ModelDefinition model, Matrix positionMatrix) {
Vector3[] positions = new Vector3[model.JointCount];
for (int jointIndex = 0; jointIndex < model.JointCount; jointIndex++) {
Vector3 newPosition = new Vector3 {
x = (float) positionMatrix[jointIndex, 0],
y = (float) positionMatrix[jointIndex, 1],
z = (float) positionMatrix[jointIndex, 2]
};
positions[jointIndex] = newPosition;
}
return positions;
}
}
}