Editor/RetargetWindow.MotionBricks.cs

Editor partial class for the RetargetWindow that adds MotionBricks AI rebuild functionality. It manages downloading/loading a MotionBricks model, running a rebuild of a MotionDocument, showing progress UI, cancelling, and exposing the rebuilt result for preview and export.

External DownloadNetworkingFile Access
🌐 Hugging Face (model download via ModelStore/ModelInstaller)
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Editor;
using HumanoidMocap.Core.Motion;
using HumanoidMocap.EditorTools.MotionBricks;
using Sandbox;

namespace HumanoidMocap.EditorTools;

/// <summary>Optional MotionBricks rebuild of a body capture (see <see cref="MotionBricksRebuild"/>): shown beside the capture
/// as BEFORE / AFTER, and exported beside it with a <c>_rebuilt</c> suffix. Needs the Humanoid Retargeter library and the
/// MotionBricks model, which is shared with AI Animator and downloaded once on request.</summary>
public sealed partial class RetargetWindow
{
    /// <summary>Added to the exported file name of the rebuilt animation.</summary>
    public const string RebuiltSuffix="_rebuilt";
    const string MotionBricksTip="Reconstruct the animation with AI (MotionBricks, NVIDIA's motion model) and compare it side by side with the capture. Key poses come from the capture; the motion between them is generated. Fingers, wrists, neck and head keep the capture. Export then writes both.";
    global::Editor.Button _motionBricksButton;
    /// <summary>The rebuilt capture and the capture it was built from; dropped when the capture changes.</summary>
    MotionDocument _rebuiltMotion,_rebuiltFrom;
    PreviewWidget _rebuiltPreview;
    BakedPreview _rebuiltBaked;
    CancellationTokenSource _rebuilding;
    static MotionBricksModel _motionBricksModel;
    static readonly SemaphoreSlim MotionBricksLoad=new(1,1);

    /// <summary>The rebuilt animation for the current capture, or null.</summary>
    MotionDocument CurrentRebuilt=>_rebuiltMotion is not null&&_rebuiltFrom==_editedMotion?_rebuiltMotion:null;
    internal PreviewWidget GateRebuiltPreview=>_rebuiltPreview;
    internal bool GateRebuiltShown=>_rebuiltPreview.IsValid()&&_rebuiltBaked is not null;
    internal string GateRebuiltNote=>_rebuiltMotion?.Diagnostics.LastOrDefault(d=>d.StartsWith("Rebuilt with MotionBricks",StringComparison.Ordinal));
    internal Task GateRebuildAsync()=>RebuildWithMotionBricksAsync(confirmDownload:false);

    void RefreshMotionBricksButton()
    {
        if(!_motionBricksButton.IsValid())return;
        // Whole-body captures only: the model has no hand-only mode.
        var body=_editedMotion?.Bones.Any(b=>b.Role==HumanoidMocap.Core.Mapping.BoneRole.UpperLegL)==true;
        _motionBricksButton.Visible=body;
        _motionBricksButton.Enabled=body&&_processing is null&&!_exportingMotion;
        var shown=CurrentRebuilt is not null;
        _motionBricksButton.Text=_rebuilding is not null?"Cancel reconstruction":shown?"Remove AI version":"Reconstruct with AI";
        _motionBricksButton.Icon=_rebuilding is not null?"cancel":shown?"close":"auto_awesome";
        _motionBricksButton.ToolTip=shown?"Remove the AI version and export only the capture.":MotionBricksTip;
    }

    void MotionBricksClicked()
    {
        if(_rebuilding is not null){_rebuilding.Cancel();return;}
        if(CurrentRebuilt is not null)
        {
            _rebuiltMotion=null;_rebuiltFrom=null;
            _captureStatus.Text="AI version removed. Export writes the capture only.";
            UpdateExportAvailability();_=RefreshMocapPreviewAsync();return;
        }
        _=RebuildWithMotionBricksAsync(confirmDownload:true);
    }

    async Task RebuildWithMotionBricksAsync(bool confirmDownload)
    {
        if(_editedMotion is null||_rebuilding is not null)return;
        var library=RetargeterLibrary.Find(out var problem);
        if(library is null){_captureStatus.Text=problem;return;}
        var store=new ModelStore(ModelPackage.MotionBricks);
        if(!store.Inspect().IsUsable&&_motionBricksModel is null)
        {
            if(!confirmDownload){await DownloadMotionBricksAsync(store);}
            else
            {
                var size=$"{ModelPackage.MotionBricks.TotalBytes/1e6:F0} MB";
                Dialog.AskConfirm(()=>_=DownloadThenRebuildAsync(store),
                    $"Reconstruct with AI needs the MotionBricks model ({size}, downloaded once from Hugging Face and shared with AI Animator). Download it now?",
                    "Download AI model","Download","Cancel");
                return;
            }
        }
        var source=_editedMotion;
        _rebuilding=new CancellationTokenSource();var token=_rebuilding.Token;
        UpdateExportAvailability();
        var rebuiltShown=false;
        try
        {
            ShowRebuildLoader("Loading the AI model…");
            var model=await LoadMotionBricksAsync(store,token);
            var progress=new Progress<float>(f=>{if(this.IsValid()&&_rebuilding is not null)ShowRebuildStatus($"Reconstructing with AI… {f*100:F0}%");});
            var result=await Task.Run(()=>MotionBricksRebuild.Rebuild(source,model,library,progress,token),token);
            await EditorPipeline.SwitchToMainThread();if(!this.IsValid())return;
            if(source!=_editedMotion){_captureStatus.Text="The capture changed while reconstructing; press Reconstruct with AI again.";return;}
            _rebuiltMotion=result.Motion;_rebuiltFrom=source;
            _captureStatus.Text=$"Reconstructed with AI in {result.Elapsed.TotalSeconds:F0} s · {result.AiShare*100:F0}% of the motion made by the AI, {result.KeyPoses} key poses from the capture{(result.AiShare<.95f?"; moves too quick for it keep the capture":"")}. Compare BEFORE and AFTER; Export writes both.";
            rebuiltShown=true;
            await RefreshMocapPreviewAsync();
        }
        catch(OperationCanceledException){await EditorPipeline.SwitchToMainThread();if(this.IsValid())_captureStatus.Text="Reconstruction cancelled. The capture is unchanged.";}
        catch(Exception e){await EditorPipeline.SwitchToMainThread();if(this.IsValid())_captureStatus.Text="AI reconstruction failed: "+e.Message;}
        finally
        {
            await EditorPipeline.SwitchToMainThread();
            _rebuilding?.Dispose();_rebuilding=null;
            if(this.IsValid()){UpdateExportAvailability();if(!rebuiltShown)RestorePreviewAfterRebuild();}
        }
    }

    async Task DownloadThenRebuildAsync(ModelStore store)
    {
        if(await DownloadMotionBricksAsync(store))await RebuildWithMotionBricksAsync(confirmDownload:false);
    }

    async Task<bool> DownloadMotionBricksAsync(ModelStore store)
    {
        _rebuilding=new CancellationTokenSource();var token=_rebuilding.Token;
        UpdateExportAvailability();
        var ready=false;
        try
        {
            ShowRebuildLoader("Downloading the AI model…");
            // This wording drives the loader's progress bar (see ProcessingIndicator).
            var progress=new Progress<InstallProgress>(p=>{if(this.IsValid())ShowRebuildStatus($"Downloading AI model · {p.Fraction*100:F0}% · {Math.Max(0,p.BytesTotal-p.BytesDone)/1e6:F0} MB left");});
            await Task.Run(()=>new ModelInstaller(store).InstallAsync(progress,token),token);
            return ready=store.Inspect().IsUsable;
        }
        catch(OperationCanceledException){await EditorPipeline.SwitchToMainThread();if(this.IsValid())_captureStatus.Text="AI model download cancelled. It resumes where it stopped next time.";return false;}
        catch(Exception e){await EditorPipeline.SwitchToMainThread();if(this.IsValid())_captureStatus.Text="AI model download failed: "+e.Message;return false;}
        finally
        {
            await EditorPipeline.SwitchToMainThread();
            _rebuilding?.Dispose();_rebuilding=null;
            if(this.IsValid()){UpdateExportAvailability();if(!ready)RestorePreviewAfterRebuild();}
        }
    }

    /// <summary>While the model downloads and the rebuild runs, the loader stands in the animation's place, as during a capture.</summary>
    void ShowRebuildLoader(string message)
    {
        if(!_loader.IsValid()||!_loader.Visible)
        {
            _targetHost.Layout.Clear(true);_mocapPreview=null;_rebuiltPreview=null;
            _loader=_targetHost.Layout.Add(new ProcessingIndicator(_targetHost),1);_loaderStatus=null;
        }
        _loader.Busy=true;ShowRebuildStatus(message);
    }
    void ShowRebuildStatus(string message)
    {
        _captureStatus.Text=message;
        if(_loader.IsValid())_loader.SetMessage(message);
    }
    /// <summary>A failed or cancelled rebuild puts the capture's preview back (keeping the status message).</summary>
    async void RestorePreviewAfterRebuild()
    {
        if(_mocapPreview.IsValid())return;
        var message=_captureStatus.Text;
        await RefreshMocapPreviewAsync();
        await EditorPipeline.SwitchToMainThread();
        if(this.IsValid())_captureStatus.Text=message;
    }

    /// <summary>One model per editor session (about 730 MB of weights, loaded in about a second).</summary>
    static async Task<MotionBricksModel> LoadMotionBricksAsync(ModelStore store,CancellationToken token)
    {
        await MotionBricksLoad.WaitAsync(token);
        try{return _motionBricksModel??=await Task.Run(()=>MotionBricksModel.Load(store.Directory,cancel:token),token);}
        finally{MotionBricksLoad.Release();}
    }

    /// <summary>The preview area: the capture alone, or BEFORE and AFTER side by side with plain labels at their top left.</summary>
    PreviewWidget AddPreviewPane(Widget host,string label,Func<Widget,PreviewWidget> create)
    {
        if(label is null)return host.Layout.Add(create(host),1);
        var pane=host.Layout.Add(new Widget(host),1);pane.Layout=Layout.Column();pane.Layout.Spacing=2;
        var text=pane.Layout.Add(new Label(label,pane){FixedHeight=18});
        text.SetStyles($"font-family: Consolas; font-size: 12px; font-weight: 700; color: {(label==AfterLabel?Theme.Green:Theme.TextLight).Hex}; padding-left: 4px;");
        return pane.Layout.Add(create(pane),1);
    }
    const string BeforeLabel="BEFORE",AfterLabel="AFTER";
}