Editor/MotionBricks/MotionBricksModel.cs

Editor class that loads and runs the MotionBricks G1 model used for motion planning and decoding. It loads weight files into WeightSet objects, constructs skeleton and representation data, runs a root planner, a pose planner and a conditioned convolution decoder to produce G1Motion transitions, and exposes timing information.

File AccessNative Interop
#nullable enable annotations

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Numerics;

namespace HumanoidMocap.EditorTools.MotionBricks;

using Vector3 = System.Numerics.Vector3;

/// <summary>
/// The loaded MotionBricks G1 model: root planner, pose planner, VQ decoder and support data.
/// Loading reads ~730 MB of F32 weights; keep one instance per editor session and
/// <see cref="Dispose"/> it to release them. Runs are thread-safe for sequential callers only.
/// </summary>
public sealed class MotionBricksModel : IDisposable
{
    public const int MinTokens = 6;
    public const int MaxTokens = 16;
    public const int FramesPerToken = 4;

    /// <summary>Component files of the bundle with their pinned tensor and parameter counts.</summary>
    public static readonly (string Component, string File, int Tensors, long Parameters)[] Components =
    {
        ("support", "support.gguf", 4, 972),
        ("vq-decoder", "vq-decoder.gguf", 51, 12437277),
        ("root", "root.gguf", 150, 34122833),
        ("pose", "pose.gguf", 209, 136588272),
    };

    private RootPlanner? _root;
    private PosePlanner? _pose;
    private ConditionedConvDecoder? _decoder;
    private float[]? _codebook;
    private readonly object _gate = new();

    public G1Skeleton Skeleton { get; }
    public MotionRepresentation Representation { get; }
    public long ParameterCount { get; }
    public TransitionTiming? LastTiming { get; private set; }

    private MotionBricksModel( G1Skeleton skeleton, MotionRepresentation representation, RootPlanner root, PosePlanner pose,
        ConditionedConvDecoder decoder, float[] codebook, long parameters )
    {
        Skeleton = skeleton;
        Representation = representation;
        _root = root;
        _pose = pose;
        _decoder = decoder;
        _codebook = codebook;
        ParameterCount = parameters;
    }

    /// <summary>Loads and validates the bundle in <paramref name="directory"/>.</summary>
    public static MotionBricksModel Load( string directory, IProgress<float>? progress = null, CancellationToken cancel = default )
    {
        var sets = new Dictionary<string, WeightSet>();
        var totalBytes = Components.Sum( c => new FileInfo( System.IO.Path.Combine( directory, c.File ) ).Length );
        long done = 0;
        foreach ( var (component, file, tensors, expectedParameters) in Components )
        {
            cancel.ThrowIfCancellationRequested();
            var path = System.IO.Path.Combine( directory, file );
            if ( !File.Exists( path ) )
                throw new FileNotFoundException( $"The model file {file} is missing.", path );
            sets[component] = WeightSet.Load( path, component, tensors, expectedParameters, cancel );
            done += new FileInfo( path ).Length;
            progress?.Report( totalBytes > 0 ? done / (float)totalBytes : 1f );
        }

        var support = sets["support"];
        var names = (support.Metadata.TryGetValue( "motionbricks.joint_names", out var n ) ? n as string : null)?.Split( ',' )
            ?? throw new InvalidDataException( "support.gguf has no joint names." );
        var neutralData = support.Shaped( "neutral_joints", 3, G1Skeleton.JointCount ).Data;
        var neutral = Enumerable.Range( 0, G1Skeleton.JointCount ).Select( j => new Vector3( neutralData[j * 3], neutralData[j * 3 + 1], neutralData[j * 3 + 2] ) ).ToArray();
        var skeleton = new G1Skeleton( names, support.Ints( "joint_parents" ), neutral );
        var representation = new MotionRepresentation( skeleton, support.Shaped( "motion_mean", 418 ).Data, support.Shaped( "motion_std", 418 ).Data );

        var vq = sets["vq-decoder"];
        var codebook = vq.Shaped( "quantizer.vq._codebook.embed", 32, 10, 8 ).Data;
        vq.Shaped( "decoder.model.6.weight", 3, 512, 413 );
        var decoder = new ConditionedConvDecoder( vq, "decoder.", 2, MotionRepresentation.LocalMotionWidth );
        var root = new RootPlanner( sets["root"] );
        var pose = new PosePlanner( sets["pose"] );
        var parameters = sets.Values.Where( s => s.Component != "support" ).Sum( s => s.ParameterCount );
        return new MotionBricksModel( skeleton, representation, root, pose, decoder, codebook, parameters );
    }

    /// <summary>
    /// Generates one transition. Returns <c>Tokens * 4</c> frames whose first four stand for the
    /// context and last four for the target, in the (canonical) frame the constraints use.
    /// Port of <c>run_transition</c> in motion-bricks.cpp.
    /// </summary>
    public G1Motion RunTransition( TransitionConstraints c, CancellationToken cancel = default )
    {
        lock ( _gate )
        {
            var root = _root ?? throw new ObjectDisposedException( nameof( MotionBricksModel ) );
            var tokens = Math.Clamp( c.Tokens, MinTokens, MaxTokens );
            var rep = Representation;
            var watch = Stopwatch.StartNew();

            var initialX = c.Context.GlobalRoot[0];
            var initialZ = c.Context.GlobalRoot[2];
            var global = new float[8 * MotionRepresentation.GlobalRootWidth];
            var local = new float[8 * MotionRepresentation.LocalRootWidth];
            var poses = new float[8 * MotionRepresentation.InternalPoseWidth];
            for ( var b = 0; b < 8; b++ )
            {
                var source = b < 4 ? c.Context : c.Target;
                var f = b % 4;
                for ( var i = 0; i < MotionRepresentation.GlobalRootWidth; i++ )
                {
                    var value = source.GlobalRoot[f * MotionRepresentation.GlobalRootWidth + i];
                    if ( i == 0 ) value -= initialX;
                    if ( i == 2 ) value -= initialZ;
                    global[b * MotionRepresentation.GlobalRootWidth + i] = rep.Normalize( value, i );
                }
                for ( var i = 0; i < MotionRepresentation.LocalRootWidth; i++ )
                    local[b * MotionRepresentation.LocalRootWidth + i] = rep.Normalize( source.LocalRoot[f * MotionRepresentation.LocalRootWidth + i], 5 + i );
                poses[b * MotionRepresentation.InternalPoseWidth] = rep.Normalize( source.GlobalRoot[f * MotionRepresentation.GlobalRootWidth + 1], 8 );
                for ( var i = 0; i < MotionRepresentation.ExternalPoseWidth; i++ )
                    poses[b * MotionRepresentation.InternalPoseWidth + 1 + i] = rep.Normalize( source.Poses[f * MotionRepresentation.ExternalPoseWidth + i], 9 + i );
            }

            var planned = root.Run( global, c.HasGlobalRoot, local, c.HasLocalRoot, poses, c.HasPoses, tokens, cancel );
            var frames = tokens * FramesPerToken;
            var rootMs = watch.Elapsed.TotalMilliseconds;

            // Local root features (heading velocity, x/z velocity, height) of the planned path.
            var rawGlobal = new float[frames * MotionRepresentation.GlobalRootWidth];
            var heading = new float[frames];
            for ( var f = 0; f < frames; f++ )
            {
                for ( var i = 0; i < MotionRepresentation.GlobalRootWidth; i++ )
                    rawGlobal[f * 5 + i] = rep.Unnormalize( planned.GlobalRoot[f * 5 + i], i );
                heading[f] = MathF.Atan2( rawGlobal[f * 5 + 4], rawGlobal[f * 5 + 3] );
            }
            var predictedLocal = new float[frames * MotionRepresentation.LocalRootWidth];
            for ( var f = 0; f < frames; f++ )
            {
                var from = f + 1 < frames ? f : f - 1;
                var to = f + 1 < frames ? f + 1 : f;
                var raw = new[]
                {
                    MotionRepresentation.WrapAngle( heading[to] - heading[from] ) * MotionRepresentation.Fps,
                    (rawGlobal[to * 5] - rawGlobal[from * 5]) * MotionRepresentation.Fps,
                    (rawGlobal[to * 5 + 2] - rawGlobal[from * 5 + 2]) * MotionRepresentation.Fps,
                    rawGlobal[f * 5 + 1],
                };
                for ( var i = 0; i < 4; i++ )
                    predictedLocal[f * 4 + i] = rep.Normalize( raw[i], 5 + i );
            }
            if ( c.HasLocalRoot[7] != 0 )
                local.AsSpan( 7 * 4, 4 ).CopyTo( predictedLocal.AsSpan( (frames - 1) * 4 ) );

            var poseCondition = new float[frames * MotionRepresentation.InternalPoseWidth];
            var hasPoseCondition = new byte[frames];
            for ( var b = 0; b < 8; b++ )
            {
                var frame = b < 4 ? b : frames - 8 + b;
                poses.AsSpan( b * MotionRepresentation.InternalPoseWidth, MotionRepresentation.InternalPoseWidth )
                    .CopyTo( poseCondition.AsSpan( frame * MotionRepresentation.InternalPoseWidth ) );
                hasPoseCondition[frame] = c.HasPoses[b];
            }
            var poseRoot = new float[frames * 4];
            for ( var f = 0; f < frames; f++ )
            {
                poseRoot[f * 4] = planned.GlobalRoot[f * 5];
                poseRoot[f * 4 + 1] = planned.GlobalRoot[f * 5 + 2];
                poseRoot[f * 4 + 2] = planned.GlobalRoot[f * 5 + 3];
                poseRoot[f * 4 + 3] = planned.GlobalRoot[f * 5 + 4];
            }

            watch.Restart();
            var codes = new int[tokens * PosePlanner.Heads];
            Array.Fill( codes, PosePlanner.MaskToken );
            var logits = _pose!.Run( codes, poseRoot, poseCondition, hasPoseCondition, tokens, cancel );
            if ( logits.Any( v => !float.IsFinite( v ) ) )
                throw new InvalidDataException( "The pose network produced invalid values." );
            codes = PosePlanner.Sample( logits, c.Seed, c.Argmax );
            var poseMs = watch.Elapsed.TotalMilliseconds;

            watch.Restart();
            var codebook = _codebook!;
            var quantized = new float[tokens * 256];
            for ( var p = 0; p < tokens; p++ )
                for ( var h = 0; h < PosePlanner.Heads; h++ )
                    for ( var d = 0; d < 32; d++ )
                        quantized[(h * 32 + d) * tokens + p] = codebook[(h * PosePlanner.Codes + codes[p * PosePlanner.Heads + h]) * 32 + d];
            var external = new float[frames * 2];
            for ( var f = 0; f < frames; f++ )
            {
                external[f * 2] = predictedLocal[f * 4 + 1];
                external[f * 2 + 1] = predictedLocal[f * 4 + 2];
            }
            var mask = hasPoseCondition.Select( v => v != 0 ? 1f : 0f ).ToArray();
            var decoded = _decoder!.Forward( quantized, 256, tokens, external, poseCondition, mask, cancel );
            var decoderMs = watch.Elapsed.TotalMilliseconds;

            watch.Restart();
            if ( decoded.Any( v => !float.IsFinite( v ) ) )
                throw new InvalidDataException( "The motion decoder produced invalid values." );
            var motion = rep.Decode( decoded, frames, initialX, initialZ );
            LastTiming = new TransitionTiming( rootMs, poseMs, decoderMs, watch.Elapsed.TotalMilliseconds );
            return motion;
        }
    }

    public void Dispose()
    {
        lock ( _gate )
        {
            _root = null;
            _pose = null;
            _decoder = null;
            _codebook = null;
        }
    }
}