Editor/Inference/FlowDenoiser.cs
// C# adaptation of FlowHMR's HunyuanMotionMMDiT. See THIRD_PARTY_NOTICES.md.
using System;
using System.Threading;

namespace HumanoidMocapX.Inference;

/// <summary>Batch-one, unpadded FlowHMR x1 predictor. Video conditioning must come
/// from SAM 3D Body and relative world-to-camera rotations. Instances are not thread safe.</summary>
internal sealed class FlowDenoiser
{
    internal const int FeatureChannels = 3072, CameraChannels = 9, TextChannels = 768;
    internal const int ModelWidth = 1024, ModelHeads = 16, DoubleBlocks = 6, SingleBlocks = 12;
    readonly int frames, width;
    readonly FlowLinear inputEncoder, featureEncoder, cameraEncoder, finalModulation, finalProjection;
    readonly FlowConditionMlp timeEncoder;
    readonly FlowRefiner refiner;
    readonly FlowDoubleBlock[] doubles;
    readonly FlowSingleBlock[] singles;
    readonly FlowBlockBuffers buffers;
    readonly FlowLayerNorm finalNorm;
    readonly float[] features, context, textCondition, timeEmbedding, adapter, activatedAdapter;
    bool hasVideo;

    internal FlowDenoiser(FlowWeights weights, int frames, int width = ModelWidth, int heads = ModelHeads)
    {
        this.frames = frames; this.width = width;
        buffers = new(frames, width, heads);
        inputEncoder = new(weights, "input_encoder", FlowSampler.Channels, width);
        featureEncoder = new(weights, "ctxt_encoder.feature", FeatureChannels, width);
        cameraEncoder = new(weights, "ctxt_encoder.camera_R", CameraChannels, width);
        timeEncoder = new(weights, "timestep_encoder.blocks", width, width);
        var textEncoder = new FlowConditionMlp(weights, "vtxt_encoder.linears", TextChannels, width);
        refiner = new(weights, frames, width, heads);
        finalModulation = new(weights, "final_layer.adaLN_modulation.linear", width, 2 * width);
        finalProjection = new(weights, "final_layer.linear", width, FlowSampler.Channels);
        finalNorm = new(width, 1e-5f);
        features = new float[2 * buffers.StreamElements]; context = new float[buffers.StreamElements];
        textCondition = new float[width]; timeEmbedding = new float[width]; adapter = new float[width]; activatedAdapter = new float[width];
        textEncoder.Run(new float[TextChannels], textCondition, default);
        doubles = new FlowDoubleBlock[DoubleBlocks]; singles = new FlowSingleBlock[SingleBlocks];
        for (var i = 0; i < doubles.Length; i++) doubles[i] = new(weights, i, width, heads);
        for (var i = 0; i < singles.Length; i++) singles[i] = new(weights, i, width, heads);
    }

    /// <summary>Binds [frames,3072] body features and [frames,9] camera rotations.
    /// Inputs are consumed immediately. Failure invalidates the previous video binding.</summary>
    internal void SetVideo(ReadOnlySpan<float> bodyFeatures, ReadOnlySpan<float> cameraRotations, CancellationToken cancellation = default)
    {
        hasVideo = false;
        cancellation.ThrowIfCancellationRequested();
        Validate(bodyFeatures, (long)frames * FeatureChannels, nameof(bodyFeatures));
        Validate(cameraRotations, (long)frames * CameraChannels, nameof(cameraRotations));
        featureEncoder.Run(bodyFeatures, frames, context, cancellation);
        var cameras = buffers.Normalized.AsSpan(0, context.Length);
        cameraEncoder.Run(cameraRotations, frames, cameras, cancellation);
        NeuralOps.AddInPlace(context, cameras);
        Validate(context, context.Length, "encoded video");
        hasVideo = true;
    }

    /// <summary>Predicts clean [frames,491] motion at time in [0,1]. Writes the output
    /// buffer without allocating; cancellation leaves partial output that must be discarded.</summary>
    internal void Predict(ReadOnlySpan<float> state, float time, Span<float> output, CancellationToken cancellation = default)
    {
        cancellation.ThrowIfCancellationRequested();
        if (!hasVideo) throw new InvalidOperationException("Bind video features before FlowHMR prediction.");
        Validate(state, (long)frames * FlowSampler.Channels, nameof(state));
        if (output.Length != state.Length || state.Overlaps(output)) throw new ArgumentException("Prediction needs a separate output tensor with the state shape.");
        if (!float.IsFinite(time) || time < 0 || time > 1) throw new ArgumentOutOfRangeException(nameof(time));
        var count = buffers.StreamElements;
        inputEncoder.Run(state, frames, features.AsSpan(0, count), cancellation);
        FlowConditioning.Time(time, 1000, timeEmbedding);
        timeEncoder.Run(timeEmbedding, adapter, cancellation); NeuralOps.AddInPlace(adapter, textCondition);
        adapter.AsSpan().CopyTo(activatedAdapter); FlowConditioning.Silu(activatedAdapter);
        refiner.Run(context, time, features.AsSpan(count), cancellation);
        foreach (var block in doubles) block.Run(features, activatedAdapter, buffers, cancellation);
        foreach (var block in singles) block.Run(features, activatedAdapter, buffers, cancellation);
        var gates = buffers.Gates.AsSpan(0, 2 * width); var normalized = buffers.Normalized.AsSpan(0, count);
        finalModulation.Run(activatedAdapter, 1, gates, cancellation);
        finalNorm.Run(features.AsSpan(0, count), normalized);
        FlowConditioning.Modulate(normalized, gates[..width], gates[width..]);
        finalProjection.Run(normalized, frames, output, cancellation);
        Validate(output, output.Length, "predicted motion");
    }

    static void Validate(ReadOnlySpan<float> values, long length, string name)
    {
        if (values.Length != length) throw new ArgumentException("FlowHMR tensor shape mismatch: " + name);
        foreach (var value in values)
            if (!float.IsFinite(value)) throw new ArithmeticException("Non-finite FlowHMR tensor: " + name);
    }
}