tests/common/observation_fixture.cs
using System.Numerics;
using HumanoidMocapX.Core.Mapping;
using HumanoidMocapX.Core.Motion;
using HumanoidMocapX.Inference;
/// <summary>SAM observations that see exactly a motion document: body, face and hand keypoints
/// from its joints, through a camera whose axes are world axes turned upside down (x right,
/// y down, z forward) at a fixed position. Hand passes and the body pass agree in the image.</summary>
static class ObservationFixture
{
internal static readonly Quaternion Camera = Quaternion.CreateFromAxisAngle(Vector3.UnitX, MathF.PI);
static readonly Vector3 CameraPosition = new(0, 1, 5);
// The template's eye-to-nose line runs 42.33° below the head's forward axis.
static readonly Vector3 Face = new(0, -MathF.Sin(42.33f * MathF.PI / 180) * .05f, MathF.Cos(42.33f * MathF.PI / 180) * .05f);
static readonly string[] Fingers = { "Thumb", "Index", "Middle", "Ring", "Pinky" };
internal static float[] Observe(MotionDocument motion)
{
var observations = new float[motion.Frames.Count * SamObservations.Elements];
for (var f = 0; f < motion.Frames.Count; f++)
{
var observation = observations.AsSpan(f * SamObservations.Elements, SamObservations.Elements);
var position = Positions(motion, motion.Frames[f], out var rotation, out var beyond);
Vector3 At(string role) => position[Bone(motion, role)];
var head = Bone(motion, "Head");
var eyes = position[head] + Vector3.Transform(new Vector3(0, .095f, .062f), rotation[head]);
var root = Camera3(At("UpperLegL") + At("UpperLegR")) / 2 - new Vector3(0, -.9f, 0);
var body = observation.Slice(SamObservations.BodyKeypoints, SamKeypoints.Count * 3);
(int Index, Vector3 World)[] keypoints =
{
(0, eyes + Vector3.Transform(Face, rotation[head])),
(1, eyes + Vector3.Transform(new Vector3(.033f, 0, 0), rotation[head])),
(2, eyes - Vector3.Transform(new Vector3(.033f, 0, 0), rotation[head])),
(5, At("UpperArmL")), (6, At("UpperArmR")), (7, At("LowerArmL")), (8, At("LowerArmR")), (9, At("UpperLegL")), (10, At("UpperLegR")),
(11, At("LowerLegL")), (12, At("LowerLegR")), (13, At("FootL")), (14, At("FootR")),
(SamObservations.RightHandStart + 20, At("HandR")), (SamObservations.LeftHandStart + 20, At("HandL"))
};
foreach (var (index, world) in keypoints) Write(body, index, Camera3(world) - root);
Write(observation, SamObservations.Translation / 3, root);
foreach (var left in new[] { true, false })
{
var hand = observation.Slice(left ? SamObservations.LeftHand : SamObservations.RightHand, SamObservations.HandPoints * 3);
var side = left ? "L" : "R";
Write(hand, 20, Camera3(At("Hand" + side)));
for (var g = 0; g < 5; g++)
{
Write(hand, g * 4 + 3, Camera3(At(Fingers[g] + "Prox" + side)));
Write(hand, g * 4 + 2, Camera3(At(Fingers[g] + "Mid" + side)));
var dist = Bone(motion, Fingers[g] + "Dist" + side);
Write(hand, g * 4 + 1, Camera3(position[dist])); Write(hand, g * 4, Camera3(beyond[dist]));
}
var image = observation.Slice(SamObservations.ImageKeypoints);
var bodyPass = image.Slice((left ? SamObservations.LeftHandStart : SamObservations.RightHandStart) * 2, SamObservations.HandPoints * 2);
var handPass = image.Slice((left ? 1 : 2) * SamKeypoints.Count * 2 + SamObservations.RightHandStart * 2, SamObservations.HandPoints * 2);
for (var i = 0; i < SamObservations.HandPoints; i++)
{
bodyPass[i * 2] = handPass[i * 2] = 400 + hand[i * 3] * 1000;
bodyPass[i * 2 + 1] = handPass[i * 2 + 1] = 300 + hand[i * 3 + 1] * 1000;
}
}
}
return observations;
}
/// <summary>One identity camera rotation per frame of <paramref name="motion"/>.</summary>
internal static float[] StaticCamera(MotionDocument motion)
{
var rotations = new float[motion.Frames.Count * 9];
for (var f = 0; f < motion.Frames.Count; f++) { rotations[f * 9] = 1; rotations[f * 9 + 4] = 1; rotations[f * 9 + 8] = 1; }
return rotations;
}
internal static int Bone(MotionDocument motion, string role) => motion.Bones.FindIndex(b => b.Role == Enum.Parse<BoneRole>(role));
/// <summary>World joint positions and rotations, and where each bone's offset would put a further joint.</summary>
internal static Vector3[] Positions(MotionDocument motion, MotionFrame frame, out Quaternion[] rotation, out Vector3[] beyond)
{
rotation = new Quaternion[motion.Bones.Count]; var position = new Vector3[rotation.Length]; beyond = new Vector3[rotation.Length];
for (var i = 0; i < rotation.Length; i++)
{
var parent = motion.Bones[i].Parent; var offset = MotionDocument.V(frame.Positions[i]);
rotation[i] = parent < 0 ? MotionDocument.Q(frame.Rotations[i]) : rotation[parent] * MotionDocument.Q(frame.Rotations[i]);
position[i] = parent < 0 ? offset : position[parent] + Vector3.Transform(offset, rotation[parent]);
beyond[i] = position[i] + Vector3.Transform(offset, rotation[i]);
}
return position;
}
static Vector3 Camera3(Vector3 world) => Vector3.Transform(world - CameraPosition, Camera);
static void Write(Span<float> points, int index, Vector3 value)
{
points[index * 3] = value.X; points[index * 3 + 1] = value.Y; points[index * 3 + 2] = value.Z;
}
}