Editor/MotionBricks/ConditionedConvDecoder.cs

An editor-side neural network decoder used by MotionBricks for token-to-frame conversion. It applies linear layers, condition injection from sparse targets and external per-frame conditions, a series of dilated residual convolutional blocks, upsampling, and returns frame-major outputs.

File Access
#nullable enable annotations

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace HumanoidMocap.EditorTools.MotionBricks;

/// <summary>
/// The convolutional token-to-frame decoder MotionBricks uses twice (root trajectory and VQ
/// pose decoding). Each token covers four frames; two stages each inject the sparse boundary
/// targets and a per-frame external condition, run a dilated residual stack (27, 9, 3, 1) and
/// upsample time 2x.
/// </summary>
public sealed class ConditionedConvDecoder
{
    private static readonly int[] Dilations = { 27, 9, 3, 1 };
    private const int Width = 512;

    private readonly WeightSet _w;
    private readonly string _prefix;
    private readonly int _externalWidth;
    private readonly int _outputWidth;

    /// <param name="prefix">Tensor name prefix, e.g. <c>"_conv_output."</c> or <c>"decoder."</c>.</param>
    public ConditionedConvDecoder( WeightSet weights, string prefix, int externalWidth, int outputWidth )
    {
        _w = weights;
        _prefix = prefix;
        _externalWidth = externalWidth;
        _outputWidth = outputWidth;
        // Resolve eagerly so a mismatched file fails at load time, not mid-generation.
        _ = weights[prefix + "model.0.weight"];
        _ = weights[prefix + "model.6.weight"];
    }

    /// <param name="tokens">Channel-major <c>[channels, positions]</c> token features.</param>
    /// <param name="external">Frame-major <c>[frames, externalWidth]</c> condition.</param>
    /// <param name="target">Frame-major <c>[frames, targetWidth]</c> sparse targets.</param>
    /// <param name="targetMask">1 where a frame has a target, else 0.</param>
    /// <returns>Frame-major <c>[frames, outputWidth]</c>.</returns>
    public float[] Forward( float[] tokens, int tokenChannels, int positions, float[] external, float[] target, float[] targetMask, CancellationToken cancel )
    {
        var frames = positions * 4;
        var hidden = Conv( tokens, tokenChannels, positions, "model.0", 1, 1 );
        NeuralOps.Relu( hidden );

        for ( var stageIndex = 0; stageIndex < 2; stageIndex++ )
        {
            cancel.ThrowIfCancellationRequested();
            var stage = stageIndex + 2;
            var group = 1 << (2 - stageIndex);
            var stagePositions = positions << stageIndex;
            var groupWidth = Width / group;

            var targetEmbedding = NeuralOps.Linear( target, frames,
                _w[$"{_prefix}target_cond_blocks.{stageIndex * 2}.weight"], _w[$"{_prefix}target_cond_blocks.{stageIndex * 2}.bias"] );
            NeuralOps.Relu( targetEmbedding );

            // [stagePositions, 512] frame-major rows are exactly [frames, 512 / group].
            var hiddenFrames = NeuralOps.Transpose( hidden, Width, stagePositions );
            for ( var f = 0; f < frames; f++ )
            {
                var mask = targetMask[f];
                if ( mask == 0f )
                    continue;
                for ( var c = 0; c < groupWidth; c++ )
                {
                    var index = f * groupWidth + c;
                    hiddenFrames[index] += (targetEmbedding[index] - hiddenFrames[index]) * mask;
                }
            }

            var groupedExternal = _externalWidth * group;
            var fusedWidth = Width + groupedExternal;
            var fused = new float[stagePositions * fusedWidth];
            for ( var p = 0; p < stagePositions; p++ )
            {
                hiddenFrames.AsSpan( p * Width, Width ).CopyTo( fused.AsSpan( p * fusedWidth ) );
                external.AsSpan( p * groupedExternal, groupedExternal ).CopyTo( fused.AsSpan( p * fusedWidth + Width ) );
            }
            var projected = NeuralOps.Linear( fused, stagePositions,
                _w[$"{_prefix}external_cond_blocks.{stageIndex * 2}.weight"], _w[$"{_prefix}external_cond_blocks.{stageIndex * 2}.bias"] );
            NeuralOps.Relu( projected );
            hidden = NeuralOps.Transpose( projected, stagePositions, Width );

            for ( var block = 0; block < Dilations.Length; block++ )
            {
                var d = Dilations[block];
                var branch = (float[])hidden.Clone();
                NeuralOps.Relu( branch );
                branch = Conv( branch, Width, stagePositions, $"model.{stage}.0.model.{block}.conv1", d, d );
                NeuralOps.Relu( branch );
                branch = Conv( branch, Width, stagePositions, $"model.{stage}.0.model.{block}.conv2", 0, 1 );
                NeuralOps.AddInPlace( hidden, branch );
            }

            hidden = NeuralOps.UpsampleNearest2x( hidden, Width, stagePositions );
            hidden = Conv( hidden, Width, stagePositions * 2, $"model.{stage}.2", 1, 1 );
        }

        hidden = Conv( hidden, Width, frames, "model.4", 1, 1 );
        NeuralOps.Relu( hidden );
        hidden = Conv( hidden, Width, frames, "model.6", 1, 1 );
        return NeuralOps.Transpose( hidden, _outputWidth, frames );
    }

    private float[] Conv( float[] input, int channels, int length, string name, int padding, int dilation )
        => NeuralOps.Conv1d( input, channels, length, _w[$"{_prefix}{name}.weight"], _w[$"{_prefix}{name}.bias"], padding, dilation );
}