Editor/Inference/HandTransfer.cs
using System;
using System.Numerics;
using HumanoidMocapX.Core.Mapping;
using HumanoidMocapX.Core.Motion;
namespace HumanoidMocapX.Inference;
using Vector3 = System.Numerics.Vector3;
/// <summary>Turns each hand to the palm orientation SAM sees in its hand close-up. FlowHMR's
/// palms face 55–89° away from SAM's on the breakin clip (25–33° on a real person) while the
/// forearms agree within about 20°, so fingers posed on them land wrong in the image. The camera
/// is related to the capture in each frame through the torso's up and shoulder line plus that
/// arm's upper arm and forearm, so the wrist keeps following the forearm even where the body is
/// turned differently; the torso keeps the fit determined when the arm lies along the shoulders.</summary>
internal static class HandTransfer
{
const double Sigma = 2;
// Frames whose arm directions cannot be matched this well say nothing reliable about the wrist.
const float FitResidual = 30 * MathF.PI / 180;
const float ForearmFull = 15 * MathF.PI / 180, ForearmNone = 35 * MathF.PI / 180;
const int LeftShoulder = 5, RightShoulder = 6, LeftElbow = 7, RightElbow = 8, LeftHip = 9, RightHip = 10;
const int LeftWrist = SamObservations.LeftHandStart + FingerTransfer.Wrist, RightWrist = SamObservations.RightHandStart + FingerTransfer.Wrist;
/// <summary>Rewrites the hand rotations of <paramref name="document"/>, whose frames match the
/// per-frame <paramref name="observations"/> one to one. Returns the share of hand-frames
/// that used observations, from 0 to 1; 0 when the arm or finger-base bones are missing.</summary>
internal static double Apply(MotionDocument document, ReadOnlySpan<float> observations)
{
var frames = document.Frames.Count;
if (frames == 0 || observations.Length != frames * SamObservations.Elements)
throw new ArgumentException("Hand transfer needs one SAM observation per motion frame.");
var world = new Quaternion[document.Bones.Count]; var position = new Vector3[world.Length];
double used = 0; var hands = 0;
foreach (var left in new[] { true, false })
{
if (Bones(document, left) is not { } bones) continue;
hands++;
var targets = new float[frames * 4]; var weights = new double[frames];
for (var f = 0; f < frames; f++)
{
MotionPose.World(document, document.Frames[f], world, position);
var target = Target(observations.Slice(f * SamObservations.Elements, SamObservations.Elements), left, bones, world, position, out var weight);
// Same hemisphere as the current rotation, so averaging neighbours cannot cancel.
var current = MotionDocument.Q(document.Frames[f].Rotations[bones.Hand]);
if (Quaternion.Dot(target, current) < 0) target = Quaternion.Negate(target);
targets[f * 4] = target.X; targets[f * 4 + 1] = target.Y; targets[f * 4 + 2] = target.Z; targets[f * 4 + 3] = target.W;
weights[f] = weight;
}
var smoothed = new float[targets.Length]; var trust = new double[frames];
MotionPose.Smooth(targets, weights, 4, Sigma, smoothed, trust);
for (var f = 0; f < frames; f++)
{
if (trust[f] <= 0) continue;
var target = new Quaternion(smoothed[f * 4], smoothed[f * 4 + 1], smoothed[f * 4 + 2], smoothed[f * 4 + 3]);
if (!(target.LengthSquared() > 1e-8f)) continue;
var original = MotionDocument.Q(document.Frames[f].Rotations[bones.Hand]);
var blended = Quaternion.Normalize(Quaternion.Slerp(original, Quaternion.Normalize(target), (float)trust[f]));
document.Frames[f].Rotations[bones.Hand] = new[] { blended.X, blended.Y, blended.Z, blended.W };
used += trust[f];
}
}
return hands == 0 ? 0 : used / (hands * frames);
}
readonly record struct HandBones(int Shoulder, int OtherShoulder, int Elbow, int Hand, int Index, int Middle, int Pinky, int HipL, int HipR);
static HandBones? Bones(MotionDocument document, bool left)
{
var bones = new HandBones(
MotionPose.Bone(document, left ? BoneRole.UpperArmL : BoneRole.UpperArmR), MotionPose.Bone(document, left ? BoneRole.UpperArmR : BoneRole.UpperArmL),
MotionPose.Bone(document, left ? BoneRole.LowerArmL : BoneRole.LowerArmR), MotionPose.Bone(document, left ? BoneRole.HandL : BoneRole.HandR),
MotionPose.Bone(document, left ? BoneRole.IndexProxL : BoneRole.IndexProxR), MotionPose.Bone(document, left ? BoneRole.MiddleProxL : BoneRole.MiddleProxR),
MotionPose.Bone(document, left ? BoneRole.PinkyProxL : BoneRole.PinkyProxR),
MotionPose.Bone(document, BoneRole.UpperLegL), MotionPose.Bone(document, BoneRole.UpperLegR));
ReadOnlySpan<int> all = stackalloc int[] { bones.Shoulder, bones.OtherShoulder, bones.Elbow, bones.Hand, bones.Index, bones.Middle, bones.Pinky, bones.HipL, bones.HipR };
foreach (var bone in all) if (bone < 0) return null;
return document.Bones[bones.Hand].Parent == bones.Elbow ? bones : null;
}
/// <summary>The hand's local rotation that puts its palm where SAM sees it, and how far to trust it.</summary>
static Quaternion Target(ReadOnlySpan<float> observation, bool left, HandBones bones, Quaternion[] world, Vector3[] position, out double weight)
{
weight = 0;
var current = MotionPose.Local(world, bones.Hand, bones.Elbow);
var hand = observation.Slice(left ? SamObservations.LeftHand : SamObservations.RightHand, SamObservations.HandPoints * 3);
var body = observation.Slice(SamObservations.BodyKeypoints, SamKeypoints.Count * 3);
if (!FingerTransfer.Palm(SamObservations.Point(hand, FingerTransfer.Wrist), SamObservations.Point(hand, FingerTransfer.IndexBase),
SamObservations.Point(hand, FingerTransfer.MiddleBase), SamObservations.Point(hand, FingerTransfer.PinkyBase), out var seenPalm)
|| !FingerTransfer.Palm(position[bones.Hand], position[bones.Index], position[bones.Middle], position[bones.Pinky], out var posedPalm))
return current;
var shoulder = SamObservations.Point(body, left ? LeftShoulder : RightShoulder); var other = SamObservations.Point(body, left ? RightShoulder : LeftShoulder);
var elbow = SamObservations.Point(body, left ? LeftElbow : RightElbow); var wrist = SamObservations.Point(body, left ? LeftWrist : RightWrist);
var hips = SamObservations.Point(body, LeftHip) + SamObservations.Point(body, RightHip);
ReadOnlySpan<Vector3> seen = stackalloc Vector3[] { shoulder + other - hips, other - shoulder, elbow - shoulder, wrist - elbow };
ReadOnlySpan<Vector3> posed = stackalloc Vector3[]
{
position[bones.Shoulder] + position[bones.OtherShoulder] - position[bones.HipL] - position[bones.HipR], position[bones.OtherShoulder] - position[bones.Shoulder],
position[bones.Elbow] - position[bones.Shoulder], position[bones.Hand] - position[bones.Elbow]
};
var toWorld = RotationFit.Fit(seen, posed, FitResidual);
var forearm = Vector3.Normalize(Vector3.Transform(seen[3], toWorld)); var posedForearm = Vector3.Normalize(posed[3]);
var miss = MathF.Acos(Math.Clamp(Vector3.Dot(forearm, posedForearm), -1f, 1f));
// The palm in the hand's own frame is fixed by the skeleton; only its world turn changes.
var palmInHand = Quaternion.Inverse(world[bones.Hand]) * Quaternion.CreateFromRotationMatrix(posedPalm);
var handWorld = toWorld * Quaternion.CreateFromRotationMatrix(seenPalm) * Quaternion.Inverse(palmInHand);
weight = FingerTransfer.Agreement(observation, left) * Math.Clamp((ForearmNone - miss) / (ForearmNone - ForearmFull), 0, 1);
return Quaternion.Normalize(Quaternion.Inverse(world[bones.Elbow]) * handWorld);
}
}