Editor/Inference/SamMhrParameters.cs
// C# adaptation of SAM 3D Body's MHR head. See LICENSE-SAM and THIRD_PARTY_NOTICES.md.
// Copyright (c) Meta Platforms, Inc. and affiliates.
using System;
using System.IO;
using System.Threading;

namespace HumanoidMocapX.Inference;

/// <summary>Maps the released SAM head's 519 predictions into MHR's 204 parameters.</summary>
internal sealed class SamMhrParameters
{
    internal const int Predictions = 519, ModelParameters = 204, PoseParameters = 136;
    internal const int ShapeOffset = 266, Shapes = 45, ScaleOffset = 311, Scales = 28,
        HandOffset = 339, FaceOffset = 447, Expressions = 72;
    readonly Tensor scale, scaleMean, hand, handMean, wrist, root, wristRotation;
    readonly int[] leftIndices, rightIndices, nonhandIndices;
    readonly bool isHand;
    readonly float[] bodyPose = new float[SamCompactPose.BodyParameters], handContinuous = new float[SamCompactPose.HandContinuous];
    readonly float[] handAngles = new float[SamCompactPose.HandParameters];

    internal SamMhrParameters(FlowWeights weights, ReadOnlySpan<int> left, ReadOnlySpan<int> right, ReadOnlySpan<int> nonhand, bool isHand = false)
    {
        this.isHand = isHand;
        var name = isHand ? "head_pose_hand." : "head_pose.";
        scale = Transpose(weights.Get(name + "scale_comps", 68, Scales)); scaleMean = weights.Get(name + "scale_mean", 68);
        hand = Transpose(weights.Get(name + "hand_pose_comps", 54, 54)); handMean = weights.Get(name + "hand_pose_mean", 54);
        wrist = weights.Get(name + "right_wrist_coords", 3); root = weights.Get(name + "root_coords", 3);
        wristRotation = weights.Get(name + "local_to_world_wrist", 3, 3);
        if (left.Length != 27 || right.Length != 27 || nonhand.Length != 145)
            throw new InvalidDataException("SAM hand parameter index counts do not match the released model.");
        Span<bool> used = stackalloc bool[ModelParameters]; used.Clear();
        ValidateIndices(left, used, PoseParameters); ValidateIndices(right, used, PoseParameters);
        used.Clear(); ValidateIndices(nonhand, used, ModelParameters);
        leftIndices = left.ToArray(); rightIndices = right.ToArray(); nonhandIndices = nonhand.ToArray();
    }

    /// <summary>Writes the model vector. Hand/jaw slots from the body prediction
    /// are suppressed; dedicated hand predictions replace the saved hand indices.</summary>
    internal void Run(ReadOnlySpan<float> prediction, Span<float> model, CancellationToken cancellation = default)
    {
        cancellation.ThrowIfCancellationRequested();
        if (prediction.Length != Predictions || model.Length != ModelParameters || prediction.Overlaps(model))
            throw new ArgumentException("SAM MHR prediction/model dimensions are invalid or overlap.");
        SamDecoder.Finite(prediction); model.Clear();
        SamRotation.GlobalZYX(prediction.Slice(0, 6), model.Slice(3, 3));
        if (isHand) TransformWrist(model);
        SamCompactPose.Body(prediction.Slice(6, SamCompactPose.BodyContinuous), bodyPose);
        bodyPose.AsSpan(62, 54).Clear(); bodyPose.AsSpan(130, 3).Clear();
        bodyPose.AsSpan(0, 130).CopyTo(model.Slice(6, 130));
        FlowLinear.Run(prediction.Slice(ScaleOffset, Scales), 1, scale, scaleMean, model.Slice(PoseParameters), cancellation);
        ReplaceHand(prediction.Slice(HandOffset, 54), leftIndices, model, cancellation);
        ReplaceHand(prediction.Slice(HandOffset + 54, 54), rightIndices, model, cancellation);
        if (isHand) foreach (var index in nonhandIndices) model[index] = 0;
        SamDecoder.Finite(model);
    }

    void ReplaceHand(ReadOnlySpan<float> prediction, ReadOnlySpan<int> indices, Span<float> model, CancellationToken cancellation)
    {
        FlowLinear.Run(prediction, 1, hand, handMean, handContinuous, cancellation);
        SamCompactPose.Hand(handContinuous, handAngles);
        for (var i = 0; i < indices.Length; i++) model[indices[i]] = handAngles[i];
    }

    void TransformWrist(Span<float> model)
    {
        Span<float> rotation = stackalloc float[9]; Span<float> world = stackalloc float[9];
        SamRotation.MatrixXYZ(model.Slice(3, 3), rotation);
        for (var row = 0; row < 3; row++)
            for (var col = 0; col < 3; col++)
                world[row * 3 + col] = rotation[row * 3] * wristRotation.Data[col]
                    + rotation[row * 3 + 1] * wristRotation.Data[3 + col] + rotation[row * 3 + 2] * wristRotation.Data[6 + col];
        SamRotation.Euler(world, model.Slice(3, 3));
        SamRotation.MatrixXYZ(model.Slice(3, 3), rotation);
        for (var row = 0; row < 3; row++)
        {
            var value = root.Data[row];
            for (var col = 0; col < 3; col++) value += rotation[row * 3 + col] * (wrist.Data[col] - root.Data[col]);
            model[row] = -value * 10;
        }
    }

    static Tensor Transpose(Tensor tensor)
        => new(tensor.Name, NeuralOps.Transpose(tensor.Data, tensor.Rows, tensor.Cols), new[] { tensor.Rows, tensor.Cols });

    static void ValidateIndices(ReadOnlySpan<int> indices, Span<bool> used, int limit)
    {
        foreach (var index in indices)
        {
            if (index < 0 || index >= limit || used[index]) throw new InvalidDataException("SAM hand parameter index is out of range or duplicated.");
            used[index] = true;
        }
    }
}