An editor UI component for importing video and running automatic mocap capture and processing. It queues uploads, invokes native worker tasks (depth prep, conversion, body/hand capture, refinement, cleaning), applies motion cleanup and joins multi-shot captures, and loads the resulting motion into the editor.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using HumanoidMocap.Core.Motion;
using Sandbox;
namespace HumanoidMocap.EditorTools;
public sealed partial class RetargetWindow
{
readonly Queue<(string Path, bool FirstPerson, double Start, double? End)> _queuedVideos = new();
string _workerMessage;
void ReceiveWorkerProgress(string message)=>Interlocked.Exchange(ref _workerMessage,message);
const int MaximumShots=16;const double MinimumShotSeconds=.5;
/// <summary>The worker's note when a capture stopped at a cut; see the worker's ShotCutDetector.</summary>
const string ShotCutPrefix="Footage cuts to another shot at ";
const string ShotsPrefix="Shots start at ";
static List<double> ShotStarts(IEnumerable<string> diagnostics)
{
var note=diagnostics.FirstOrDefault(d=>d.StartsWith(ShotsPrefix,StringComparison.Ordinal));
if(note is null)return new List<double>();
return note.Substring(ShotsPrefix.Length).TrimEnd('.',' ','s').Split(',').Select(v=>double.TryParse(v.Trim(),System.Globalization.NumberStyles.Float,System.Globalization.CultureInfo.InvariantCulture,out var t)?t:double.NaN).Where(double.IsFinite).ToList();
}
static double? NextShotStart(IEnumerable<string> diagnostics)
{
var note=diagnostics.FirstOrDefault(d=>d.StartsWith(ShotCutPrefix,StringComparison.Ordinal));
if(note is null)return null;
var text=note.Substring(ShotCutPrefix.Length);var end=text.IndexOf(" s",StringComparison.Ordinal);
return end>0&&double.TryParse(text[..end],System.Globalization.NumberStyles.Float,System.Globalization.CultureInfo.InvariantCulture,out var seconds)?seconds:null;
}
/// <summary>One shot: the capture, then world-relative refinement when its camera allows. A camera the
/// worker measured as still gets world-relative root and foot-contact refinement; one whose rotation it
/// followed gets the same from GVHMR's world rollout; one that could not be followed stays camera-relative.
/// The untouched capture stays beside it and Advanced → Restore original capture reopens it.</summary>
/// <returns>The shot's motion and the capture's own notes (refinement does not keep all of them).</returns>
async Task<(string Path,List<string> Notes)> CaptureBodyShotAsync(string video,double start,double end,Mp4Metadata metadata,float? recordedFov,CancellationToken token,string depth=null,int phase=0)
{
var bodyPath=await NativeCapture.BodyAsync(video,start,end,metadata.Width,metadata.Height,recordedFov,ReceiveWorkerProgress,token,depth,phase);
var diagnostics=await Task.Run(()=>MotionDocument.Parse(File.ReadAllBytes(bodyPath)).Diagnostics,token);
if(diagnostics.Any(d=>d.StartsWith(StationaryCameraPrefix,StringComparison.Ordinal)))
return (await NativeCapture.RefineBodyAsync(bodyPath,ReceiveWorkerProgress,token),diagnostics);
if(diagnostics.Any(d=>d.StartsWith(FollowedCameraPrefix,StringComparison.Ordinal)))
return (await NativeCapture.RefineBodyAsync(bodyPath,ReceiveWorkerProgress,token,followedCameraRotation:true),diagnostics);
// Left relative to its camera: turn it upright with the network's gravity (see CaptureLevel).
return (await Task.Run(()=>
{
var document=MotionDocument.Parse(File.ReadAllBytes(bodyPath));
if(CaptureLevel.Apply(document)<=0)return bodyPath;
var levelled=Path.Combine(Path.GetDirectoryName(bodyPath),"levelled-body.hmotion");
File.WriteAllText(levelled,document.ToJson());return levelled;
},token),diagnostics);
}
internal bool CaptureIsRunning => _processing is not null;
internal string CaptureMotionPath => _motionPath;
internal string CaptureMessage => _captureStatus.Text;
internal bool CaptureCanExport => _exportMotionButton.Enabled;
internal bool CapturePreviewReady => _mocapPreview.IsValid() && _mocapPreview.FrameCount > 0 && _mocapPreview.HasModel;
internal bool CaptureWorkspaceFirstPerson => _firstPerson;
internal bool CapturePreviewFirstPerson => _mocapPreview.IsValid() && _mocapPreview.FirstPerson;
internal PreviewWidget CapturePreview => _mocapPreview;
internal void CancelCapture() => _processing?.Cancel();
public void ImportVideoAndProcess(string path) => ImportVideoAndProcess(path,0,null);
/// <summary>Process an explicitly selected range; the ordinary upload processes the full video.</summary>
public void ImportVideoAndProcess(string path, double start, double? end)
{
if (_processing is not null)
{
if (_queuedVideos.Count >= 8) { _captureStatus.Text = "Upload saved. Eight videos are already queued; open this video after processing finishes."; return; }
_queuedVideos.Enqueue((path, _firstPerson,start,end));
_captureStatus.Text = $"Video saved · {_queuedVideos.Count} queued.";
return;
}
ResetClipSpeed();
LoadVideo(path);
_rangeStart.Text = start.ToString(System.Globalization.CultureInfo.InvariantCulture);
_rangeEnd.Text = end?.ToString(System.Globalization.CultureInfo.InvariantCulture)??"";
_ = ProcessImportedVideoAsync();
}
async Task ProcessImportedVideoAsync()
{
if (_processing is not null) return;
if (!File.Exists(_sourcePath.Text)) { _captureStatus.Text = "Upload a video first."; return; }
var video = _sourcePath.Text; var firstPerson = _firstPerson;var handBackend=_handBackend;
var recordingFovText=_recordingFov.Text;
float? recordingFov=null;
if(firstPerson&&handBackend!="mediapipe"&&!string.IsNullOrWhiteSpace(recordingFovText))
{
if(!float.TryParse(recordingFovText,System.Globalization.NumberStyles.Float,System.Globalization.CultureInfo.InvariantCulture,out var degrees)||
!float.IsFinite(degrees)||degrees<20||degrees>150)
{_captureStatus.Text="Recording FOV must be a number from 20 to 150 degrees, or blank for automatic.";return;}
recordingFov=degrees;
}
var start = Number(_rangeStart, 0); double? end = string.IsNullOrWhiteSpace(_rangeEnd.Text) ? null : Number(_rangeEnd, 0);
_processing = new CancellationTokenSource(); var token = _processing.Token;
SetCaptureBusy(true); _retryCaptureButton.Visible = false;
_exportMotionButton.Enabled = false; _rawMotion = null; _editedMotion = null; _motionPath = null;
_editSession=null;_appliedCleanup=null;
++_motionLoadRevision;InvalidateMocapPreview();
_targetHost.Layout.Clear(true);_mocapPreview=null;
_loader=_targetHost.Layout.Add(new ProcessingIndicator(_targetHost),1);_loader.SetMessage("Preparing video…");_loaderStatus=null;
ResetPlayback();RefreshContacts();_motionDetails.Text="";
try
{
_captureStatus.Text = "Preparing video…";
// The lens the camera recorded (iPhones write it), read before any conversion drops the metadata.
// A lens typed under Advanced still wins.
float? recordedFov=null;
try{recordedFov=await Task.Run(()=>Mp4Metadata.Read(video).HorizontalFov,token);}catch(FormatException){}catch(IOException){}
if(recordedFov is float lensFov&&!(lensFov>=20&&lensFov<=150))recordedFov=null;
if(firstPerson&&recordingFov is null)recordingFov=recordedFov;
// Depth recorded with the video (a LiDAR app's RGBD video or .r3d, or Cinematic mode) is split from the
// picture first: an .r3d is not a video at all. Body captures use it; without it nothing changes.
var depthSource=_depthSource??await Task.Run(()=>HumanoidMocap.EditorTools.Inference.DepthDetection.DetectWithPicture(video),token);
string depthTrack=null,depthNote=null;
// A Record3D recording is split even for first person: its picture is half of each frame, or not a video at all.
if(depthSource.Kind!=HumanoidMocap.EditorTools.Inference.DepthKind.None&&(!firstPerson||depthSource.Format!=HumanoidMocap.EditorTools.Inference.DepthDetection.Cinematic))
{
try
{
// Already read for the preview (and the preview already shows its picture): reuse it.
var prepared=_depthPrepared is { } ready&&(ready.Video==video||_videoName.ToolTip==video)?ready:await NativeCapture.PrepareDepthAsync(video,ReceiveWorkerProgress,token);
_depthPrepared=prepared;
await EditorPipeline.SwitchToMainThread(); if (!this.IsValid()) return;
if(!firstPerson){depthTrack=prepared.Depth;depthNote=prepared.Note;}
if(prepared.HorizontalFov is float depthLens&&string.IsNullOrWhiteSpace(recordingFovText))recordedFov=depthLens;
if(prepared.Video!=video)
{
var original=video;
if(_sourcePath.Text!=prepared.Video){var fov=_recordingFov.Text;LoadVideo(prepared.Video,depthSource);_recordingFov.Text=fov;_videoName.Text=Path.GetFileName(original);_videoName.ToolTip=original;}
video=prepared.Video;
}
}
catch(OperationCanceledException){throw;}
catch(Exception e){depthNote="The recording's depth could not be read ("+e.Message+"); captured from the picture alone.";}
}
// Footage Windows cannot decode (iPhone HEVC, 10-bit exports) is converted once and used in its place.
var playable=await NativeCapture.PlayableVideoAsync(video,ReceiveWorkerProgress,token);
await EditorPipeline.SwitchToMainThread(); if (!this.IsValid()) return;
if(playable!=video)
{
var fov=_recordingFov.Text;LoadVideo(playable,depthSource);_recordingFov.Text=fov;
_videoName.Text=Path.GetFileName(video)+" · converted";_videoName.ToolTip=video;video=playable;
}
var metadata = await Task.Run(() => Mp4Metadata.Read(video), token);
await EditorPipeline.SwitchToMainThread(); if (!this.IsValid()) return;
_lastCaptureStride=metadata.CaptureStride;
// A long video is not refused: the first 1,800 captured frames (about a minute) are processed
// and the range shows it, so another part can be chosen under Advanced.
string lengthNote=null;
var available=metadata.CaptureTimes.Where(t=>t>=start).ToArray();
if (available.Length > 1800 && !end.HasValue)
{
end=available[1800];
_rangeEnd.Text=end.Value.ToString("0.###",System.Globalization.CultureInfo.InvariantCulture);
lengthNote=FormattableString.Invariant($"Long video: captured {start:0.#}–{end.Value:0.#} s of {metadata.Duration:0.#} s (1,800 frames at a time). Set another range under Advanced to capture a later part.");
}
string motionPath;float? pictureSpeed=null;
if (firstPerson && handBackend=="mediapipe")
{
if (!File.Exists(_handModelPath))
{
_captureStatus.Text = "Preparing hand capture · downloading the hand model…";
_handModelPath = await HandModelStore.EnsureAsync(token);
await EditorPipeline.SwitchToMainThread(); if (!this.IsValid()) return;
}
motionPath = await HandCaptureJob.RunAsync(video, _handModelPath, _target.Spec.Rig, start, end, _swapHands,
(done, total, message) => { if (this.IsValid()) _captureStatus.Text = message; }, token);
}
else if(firstPerson)
{
motionPath=await NativeCapture.HandsAsync(video,handBackend,start,end??metadata.Duration,metadata.Width,metadata.Height,
ReceiveWorkerProgress,token,recordingFov);
}
else
{
// Edited footage: the worker captures up to the first cut. Every following shot is captured
// the same way, each with its own camera, and the shots are joined into one motion.
var shotPaths=new List<string>();double shotEnd=end??metadata.Duration;var shotFailures=new List<string>();
var (firstShot,firstNotes)=await CaptureBodyShotAsync(video,start,shotEnd,metadata,recordedFov,token,depthTrack);shotPaths.Add(firstShot);
pictureSpeed=ClipSpeed.FromNotes(firstNotes);
// The worker lists every shot's start once; each shot is then captured up to the next one.
var starts=ShotStarts(firstNotes);
var queue=new List<(double Start,double End)>();
if(starts.Count>1)for(var i=1;i<starts.Count;i++)queue.Add((starts[i],i+1<starts.Count?starts[i+1]:shotEnd));
else if(NextShotStart(firstNotes) is double next&&next>start)queue.Add((next,shotEnd));
// The first capture skips a sliver of a shot at its start (a one-frame flicker read as a cut) and so
// may already cover the next listed shot.
var firstEnd=await Task.Run(()=>MotionDocument.Parse(File.ReadAllBytes(firstShot)).Frames[^1].Time,token);
queue.RemoveAll(q=>q.End-q.Start<MinimumShotSeconds||q.Start<firstEnd);
for(var i=0;i<queue.Count&&i<MaximumShots-1;i++)
{
var (shotStart,shotStop)=queue[i];
ReceiveWorkerProgress(FormattableString.Invariant($"Capturing shot {i+2} of {queue.Count+1}, from {shotStart:0.00} s"));
try{shotPaths.Add((await CaptureBodyShotAsync(video,shotStart,shotStop,metadata,recordedFov,token,depthTrack)).Path);}
catch(OperationCanceledException){throw;}
// A shot the capture cannot follow (a close-up of the legs, nobody in view) is left out.
catch(Exception e){shotFailures.Add(FormattableString.Invariant($"The shot from {shotStart:0.00} s was left out: {e.Message}"));}
}
string shotFailure=shotFailures.Count>0?string.Join(" ",shotFailures):null;
motionPath=shotPaths[0];
// A fast clip recorded at a high frame rate, to be followed at every frame: the frames in between are captured as a
// second pass and the two joined (single-shot clips; edited footage keeps its usual capture).
if(_everyFrame&&metadata.CaptureStride==2&&shotPaths.Count==1&&shotFailure is null)
{
ReceiveWorkerProgress("Capturing the frames in between, for every frame of the fast clip");
var (secondShot,_)=await CaptureBodyShotAsync(video,start,shotEnd,metadata,recordedFov,token,depthTrack,phase:1);
var firstPath=motionPath;
motionPath=await Task.Run(()=>
{
var first=MotionDocument.Parse(File.ReadAllBytes(firstPath));var second=MotionDocument.Parse(File.ReadAllBytes(secondShot));
// Passes that ended up in different spaces (one refined, one not) cannot be joined; the first stands.
if(first.Space!=second.Space)return firstPath;
var merged=MotionInterleave.Merge(first,second);
var destination=Path.Combine(Path.GetDirectoryName(firstPath),"every-frame.hmotion");
File.WriteAllText(destination,merged.ToJson());return destination;
},token);
}
if(shotPaths.Count>1||shotFailure is not null)
{
var paths=shotPaths.ToArray();var failure=shotFailure;
motionPath=await Task.Run(()=>
{
var documents=paths.Select(p=>MotionDocument.Parse(File.ReadAllBytes(p))).ToList();
// Only the last shot still ends at a cut that was not followed.
for(var i=0;i<documents.Count-1;i++)documents[i].Diagnostics.RemoveAll(d=>d.StartsWith(ShotCutPrefix,StringComparison.Ordinal));
foreach(var document in documents)document.Diagnostics.RemoveAll(d=>d.StartsWith(ShotsPrefix,StringComparison.Ordinal));
var joined=MotionJoin.Join(documents);joined.OriginalReconstruction=null;
if(failure is not null)joined.Diagnostics.Add(failure);
var destination=Path.Combine(Path.GetDirectoryName(paths[0]),"joined-shots.hmotion");
File.WriteAllText(destination,joined.ToJson());return destination;
},token);
}
}
token.ThrowIfCancellationRequested();
await EditorPipeline.SwitchToMainThread(); if (!this.IsValid()) return;
Interlocked.Exchange(ref _workerMessage,null);
_captureStatus.Text = "Cleaning motion…";
var cleanup = new CleanupSettings { Root = Number(_rootSmooth, .25f), Arms = Number(_armSmooth, .1f), Fingers = Number(_fingerSmooth, .025f) };
cleanup.Smoothing = cleanup.PositionSmoothing = SmoothingStrength();
// Body captures: foot contacts from UnderPressure replace the body network's stationary-foot guesses,
// which miss exactly the planted feet that slide. Without the model the capture keeps those guesses.
string contactModel=null;string contactNote=null;
if(!firstPerson)
{
try{contactModel=await ContactModelStore.EnsureAsync(token);}
catch(OperationCanceledException){throw;}
catch(Exception e){contactNote="Foot contacts: the UnderPressure model could not be downloaded ("+e.Message+"); the body network's stationary-foot predictions are used instead.";}
}
var cleanedPath = await Task.Run(() =>
{
var raw = MotionDocument.Parse(File.ReadAllBytes(motionPath));
// GVHMR already has a temporal model; do not stack generic cleanup on it.
var cleaned = firstPerson ? MotionCleanup.Apply(raw, cleanup) : MotionCleanup.SmoothBody(MotionCleanup.RemoveSpikes(MotionCleanup.BridgeDoubtfulFrames(raw).Motion).Motion, cleanup.Smoothing);
if(!firstPerson&&LimbHinges.Apply(cleaned) is {Changed:>0} hinges)
cleaned.Diagnostics.Add(FormattableString.Invariant($"Elbows and knees kept bending on their hinge: {hinges.Changed} limb frames bent more than {LimbHinges.ToleranceDegrees:F0} degrees sideways had the upper arm or thigh rolled instead (up to {hinges.LargestDegrees:F0} degrees); no joint moved."));
if(!firstPerson&&FingerLimits.Apply(cleaned) is var fingerSamples&&fingerSamples>0)
cleaned.Diagnostics.Add(FormattableString.Invariant($"Fingers held within their joint limits: {fingerSamples} finger joint samples bent backwards or sideways past what fingers can do, or turned about themselves, were brought back."));
if(lengthNote is not null)cleaned.Diagnostics.Add(lengthNote);
if(depthNote is not null)cleaned.Diagnostics.Add(depthNote);
if(contactNote is not null)cleaned.Diagnostics.Add(contactNote);
if(contactModel is not null&&new HumanoidMocap.EditorTools.Inference.UnderPressureContacts(contactModel).Estimate(cleaned,token) is { } contacts)
{
cleaned.StationaryJoints.RemoveAll(t=>contacts.ContainsKey(t.Bone));
foreach(var (bone,values) in contacts)cleaned.StationaryJoints.Add(new StationaryJointTrack{Bone=bone,Source=HumanoidMocap.EditorTools.Inference.UnderPressureContacts.Source,Probability=values});
cleaned.Diagnostics.Add("Foot contacts: "+HumanoidMocap.EditorTools.Inference.UnderPressureContacts.Source+". Planted heels and toes are pinned to the floor while in contact.");
}
token.ThrowIfCancellationRequested();
var destination = Path.Combine(Path.GetDirectoryName(motionPath), "automatic.edited.hmotion");
File.WriteAllText(destination, cleaned.ToJson()); return destination;
}, token);
// Planted feet held in place over the whole clip. Optional: without it the retarget's own foot lock still runs.
if(contactModel is not null)
{
await EditorPipeline.SwitchToMainThread();if(!this.IsValid())return;
_captureStatus.Text="Cleaning foot contacts…";
try{await NativeCapture.CleanFootContactsAsync(cleanedPath,contactModel,ReceiveWorkerProgress,token);}
catch(OperationCanceledException){throw;}
catch(Exception){}
}
await LoadMotionAsync(cleanedPath,motionPath,firstPerson?cleanup:null);
if(!firstPerson)await OfferSlowDownAsync(video,metadata.CaptureStride,pictureSpeed);
}
catch (OperationCanceledException)
{
await EditorPipeline.SwitchToMainThread();
if (this.IsValid()) { _captureStatus.Text = "Cancelled. Retry resumes cached reconstruction."; _retryCaptureButton.Visible = true; }
_queuedVideos.Clear();
}
catch (Exception e)
{
await EditorPipeline.SwitchToMainThread();
if (this.IsValid()) { _captureStatus.Text = e.Message; _retryCaptureButton.Visible = true; }
}
finally
{
Interlocked.Exchange(ref _workerMessage,null);
await EditorPipeline.SwitchToMainThread(); _processing.Dispose(); _processing = null;
if (this.IsValid())
{
SetCaptureBusy(false);
StartNextQueuedVideo();
}
}
}
async Task ReinstallWorkerAsync()
{
if(_processing is not null){_captureStatus.Text="Finish or cancel the active capture before reinstalling the worker.";return;}
_processing=new CancellationTokenSource();var token=_processing.Token;
SetCaptureBusy(true);_retryCaptureButton.Visible=false;_captureStatus.Text="Reinstalling the inference worker…";
try
{
await NativeCapture.ReinstallWorkerAsync(ReceiveWorkerProgress,token);
await EditorPipeline.SwitchToMainThread();
if(this.IsValid())_captureStatus.Text="Inference worker reinstalled.";
}
catch(OperationCanceledException)
{
await EditorPipeline.SwitchToMainThread();
if(this.IsValid())_captureStatus.Text="Reinstall cancelled. The worker installs again with the next capture.";
}
catch(Exception e)
{
await EditorPipeline.SwitchToMainThread();
if(this.IsValid())_captureStatus.Text=e.Message;
}
finally
{
Interlocked.Exchange(ref _workerMessage,null);
await EditorPipeline.SwitchToMainThread();_processing.Dispose();_processing=null;
if(this.IsValid())SetCaptureBusy(false);
}
}
void SetCaptureBusy(bool busy)
{
_uploadVideoButton.Enabled = !busy; _workspacePicker.Enabled = !busy; if(!busy)_workspacePicker.Visible = true;
if(_loader.IsValid()){_loader.Busy=busy;_loaderStatus=null;}
_cancelCaptureButton.Visible = busy; _advancedPanel.Enabled = !busy;
_previewTargetPicker.Enabled=!busy;
UpdateExportAvailability();
}
void StartNextQueuedVideo()
{
if(_queuedVideos.TryDequeue(out var queued))
{SetWorkspace(queued.FirstPerson);ImportVideoAndProcess(queued.Path,queued.Start,queued.End);}
}
}