InferenceWorker/ManoDecoder.cs
using System.Numerics;
using HumanoidMocap.Inference;
namespace HumanoidMocap.Worker;
/// <summary>MANO shape/pose blend shapes, joint regression, forward kinematics and
/// linear blend skinning from checkpoint buffers. All coordinates remain in the
/// backend's camera convention (metres, x-right/y-down/z-forward).</summary>
public sealed class ManoDecoder
{
const int VertexCount=778,JointCount=16;
readonly float[] template,shapeDirections,poseDirections,regressor,skinWeights,poseMean;
readonly int[] parents,tips;
public int[] Parents=>(int[])parents.Clone();
public sealed record DecodedHand(Vector3[] Joints,Vector3[] Vertices,Vector3[] RestJoints,
Quaternion[] LocalRotations,Vector3[] Landmarks);
/// <summary>The caller verifies the pinned checkpoint hash before constructing this decoder.</summary>
public ManoDecoder(TorchCheckpoint checkpoint,string prefix,bool mobileHand=false)
{
float[] Read(string name,int count)
{
var key=mobileHand?name switch{"v_template"=>"V_","shapedirs"=>"S_","posedirs"=>"P_","J_regressor"=>"J_","lbs_weights"=>"W_","parents"=>"K_",_=>name}:name;
var data=checkpoint.ReadFloat(prefix+key);
if(data.Length!=count||data.Any(v=>!float.IsFinite(v)))throw new InvalidDataException("Invalid MANO buffer: "+name);
return data;
}
template=Read("v_template",VertexCount*3);shapeDirections=Read("shapedirs",VertexCount*3*10);
poseDirections=Read("posedirs",135*VertexCount*3);regressor=Read("J_regressor",JointCount*VertexCount);
skinWeights=Read("lbs_weights",VertexCount*JointCount);poseMean=mobileHand?new float[48]:Read("pose_mean",48);
parents=Read("parents",JointCount).Select(v=>checked((int)v)).ToArray();
tips=Read(mobileHand?"fingertip_vert":checkpoint.Tensors.ContainsKey(prefix+"extra_joints_idxs")?"extra_joints_idxs":"vertex_joint_selector.extra_joints_idxs",5).Select(v=>checked((int)v)).ToArray();
if(parents[0]!=-1||parents.Skip(1).Where((p,i)=>p<0||p>i).Any()||tips.Any(i=>i<0||i>=VertexCount))
throw new InvalidDataException("Invalid MANO hierarchy or fingertip indices.");
}
/// <param name="addPoseMean">WildHands adds MANO's axis-angle mean; WiLoR's
/// MANOLayer accepts final rotation matrices directly and must not add it.</param>
public DecodedHand Decode(float[] rotationMatrices,float[] shape,bool addPoseMean,CancellationToken cancellation=default)
{
if(rotationMatrices.Length!=JointCount*9||shape.Length!=10||rotationMatrices.Concat(shape).Any(v=>!float.IsFinite(v)))
throw new ArgumentException("Invalid MANO prediction.");
var vertices=new Vector3[VertexCount];
for(var v=0;v<VertexCount;v++)
{
var x=template[v*3];var y=template[v*3+1];var z=template[v*3+2];
for(var b=0;b<10;b++)
{x+=shapeDirections[v*30+b]*shape[b];y+=shapeDirections[v*30+10+b]*shape[b];z+=shapeDirections[v*30+20+b]*shape[b];}
vertices[v]=new(x,y,z);
}
var rest=new Vector3[JointCount];
for(var j=0;j<JointCount;j++)for(var v=0;v<VertexCount;v++)rest[j]+=regressor[j*VertexCount+v]*vertices[v];
var rotations=new Quaternion[JointCount];var feature=new float[135];
for(var j=0;j<JointCount;j++)
{
cancellation.ThrowIfCancellationRequested();var i=j*9;
// System.Numerics uses row-vector matrices; checkpoints use column vectors.
var matrix=new Matrix4x4(rotationMatrices[i],rotationMatrices[i+3],rotationMatrices[i+6],0,
rotationMatrices[i+1],rotationMatrices[i+4],rotationMatrices[i+7],0,
rotationMatrices[i+2],rotationMatrices[i+5],rotationMatrices[i+8],0,0,0,0,1);
var q=Quaternion.Normalize(Quaternion.CreateFromRotationMatrix(matrix));
if(addPoseMean)
{
if(q.W<0)q=new(-q.X,-q.Y,-q.Z,-q.W);
var sine=new Vector3(q.X,q.Y,q.Z).Length();
var aa=sine<1e-8f?new Vector3(q.X,q.Y,q.Z)*2:new Vector3(q.X,q.Y,q.Z)*(2*MathF.Atan2(sine,q.W)/sine);
aa+=new Vector3(poseMean[j*3],poseMean[j*3+1],poseMean[j*3+2]);
var angle=aa.Length();q=angle<1e-8f?Quaternion.Identity:Quaternion.CreateFromAxisAngle(aa/angle,angle);
}
rotations[j]=q;
if(j==0)continue;
matrix=Matrix4x4.CreateFromQuaternion(q);var at=(j-1)*9;
feature[at]=matrix.M11-1;feature[at+1]=matrix.M21;feature[at+2]=matrix.M31;
feature[at+3]=matrix.M12;feature[at+4]=matrix.M22-1;feature[at+5]=matrix.M32;
feature[at+6]=matrix.M13;feature[at+7]=matrix.M23;feature[at+8]=matrix.M33-1;
}
for(var v=0;v<VertexCount;v++)
{
if((v&127)==0)cancellation.ThrowIfCancellationRequested();
float dx=0,dy=0,dz=0;
for(var p=0;p<135;p++)
{var i=p*VertexCount*3+v*3;dx+=feature[p]*poseDirections[i];dy+=feature[p]*poseDirections[i+1];dz+=feature[p]*poseDirections[i+2];}
vertices[v]+=new Vector3(dx,dy,dz);
}
var joints=new Vector3[JointCount];var world=new Quaternion[JointCount];
for(var j=0;j<JointCount;j++)
{
var parent=parents[j];
world[j]=parent<0?rotations[j]:Quaternion.Normalize(world[parent]*rotations[j]);
joints[j]=parent<0?rest[j]:joints[parent]+Vector3.Transform(rest[j]-rest[parent],world[parent]);
}
var skinned=new Vector3[VertexCount];
for(var v=0;v<VertexCount;v++)for(var j=0;j<JointCount;j++)
skinned[v]+=skinWeights[v*JointCount+j]*(joints[j]+Vector3.Transform(vertices[v]-rest[j],world[j]));
var all=new Vector3[21];Array.Copy(joints,all,JointCount);
for(var i=0;i<tips.Length;i++)all[16+i]=skinned[tips[i]];
int[] map={0,13,14,15,16,1,2,3,17,4,5,6,18,10,11,12,19,7,8,9,20};
return new(joints,skinned,rest,rotations,map.Select(i=>all[i]).ToArray());
}
}