Editor/Inference/SamDecoder.cs
// C# adaptation of SAM 3D Body's promptable decoder. See LICENSE-SAM and THIRD_PARTY_NOTICES.md.
// Copyright (c) Meta Platforms, Inc. and affiliates.
using System;
using System.Threading;

namespace HumanoidMocapX.Inference;

/// <summary>Predicts an intermediate pose from normalized tokens and updates the
/// keypoint tokens/positions in place. Called after layers 0–4; never retain these spans.</summary>
internal delegate void SamPoseFeedback(int layer, ReadOnlySpan<float> normalized, Span<float> tokens,
    Span<float> positions, CancellationToken cancellation);

/// <summary>Six-layer body/hand decoder with required intermediate pose feedback.
/// Inputs already include camera conditioning and prompt embeddings.</summary>
internal sealed class SamDecoder
{
    readonly SamDecoderShape shape;
    readonly SamDecoderBlock[] layers = new SamDecoderBlock[SamDecoderShape.Layers];
    readonly SamLayerNorm finalNorm;
    readonly SamDecoderWorkspace workspace;
    readonly float[] tokens, positions, normalized;

    internal SamDecoder(FlowWeights weights, SamDecoderShape shape, bool hand = false)
    {
        if (shape.Tokens == 0) throw new ArgumentException("Construct a valid SAM decoder shape.", nameof(shape));
        this.shape = shape; var name = hand ? "decoder_hand" : "decoder";
        for (var layer = 0; layer < layers.Length; layer++) layers[layer] = new(weights, name + ".layers." + layer, shape);
        finalNorm = new(weights, name + ".norm_final", shape.Width); workspace = new(shape);
        tokens = new float[shape.TokenElements]; positions = new float[shape.TokenElements]; normalized = new float[shape.TokenElements];
    }

    /// <summary>Writes all final normalized tokens, including the first pose token.
    /// The caller predicts the final pose from this output separately. Intermediate
    /// feedback is mandatory; no unconditioned fallback is supplied. Reuse sequentially.</summary>
    internal void Run(ReadOnlySpan<float> input, ReadOnlySpan<float> image, ReadOnlySpan<float> tokenPositions,
        ReadOnlySpan<float> imagePositions, Span<float> output, SamPoseFeedback feedback, CancellationToken cancellation = default)
    {
        cancellation.ThrowIfCancellationRequested(); ArgumentNullException.ThrowIfNull(feedback);
        Validate(shape, input, image, tokenPositions, imagePositions, output);
        input.CopyTo(tokens); tokenPositions.CopyTo(positions);
        for (var layer = 0; layer < layers.Length; layer++)
        {
            layers[layer].Run(tokens, image, positions, imagePositions, layer == 0, workspace, cancellation);
            finalNorm.Run(tokens, normalized); Finite(normalized);
            if (layer == layers.Length - 1) break;
            feedback(layer, normalized, tokens, positions, cancellation); Finite(tokens); Finite(positions);
        }
        cancellation.ThrowIfCancellationRequested(); normalized.AsSpan().CopyTo(output);
    }

    internal static void Validate(SamDecoderShape shape, ReadOnlySpan<float> input, ReadOnlySpan<float> image,
        ReadOnlySpan<float> tokenPositions, ReadOnlySpan<float> imagePositions, Span<float> output)
    {
        if (input.Length != shape.TokenElements || tokenPositions.Length != input.Length || output.Length != input.Length
            || image.Length != shape.ImageElements || imagePositions.Length != image.Length || input.Overlaps(output)
            || tokenPositions.Overlaps(output) || image.Overlaps(output) || imagePositions.Overlaps(output))
            throw new ArgumentException("SAM decoder tensors have invalid dimensions or overlap.");
        Finite(input); Finite(image); Finite(tokenPositions); Finite(imagePositions);
    }

    internal static void Finite(ReadOnlySpan<float> values)
    { foreach (var value in values) if (!float.IsFinite(value)) throw new ArgumentException("Non-finite SAM decoder feature or pose feedback."); }
}