InferenceWorker/BodyRefinement.cs
using System.Diagnostics;
using System.Numerics;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using HumanoidMocap.Inference;
using HumanoidMocap.Motion;
namespace HumanoidMocap.Worker;
/// <param name="UseCameraRotation">Use the camera rotation the capture followed from the
/// background (camera-rotation.json beside the capture) instead of assuming a still camera.</param>
public sealed record BodyRefinementRequest(string Motion,string Models,string Output,bool AssumeStationaryCamera=false,bool UseCameraRotation=false);
/// <summary>Cheap, reversible processing of saved GVHMR predictions. Never loads
/// the image/temporal networks or rewrites the original reconstruction.</summary>
public static class BodyRefinement
{
public const string Version="gvhmr-stationary-contact-ccd-v7";
public const string MovingVersion="gvhmr-followed-rotation-contact-ccd-v4";
/// <summary>Per-frame GVHMR camera angular velocity, and whether the camera only turned in place.</summary>
public sealed record CameraRotation(float[] AngularVelocity6d,bool RotationOnly);
public const string CameraRotationFile="camera-rotation.json";
public static string Run(BodyRefinementRequest request,CancellationToken cancellation,Action<string>? progress=null)
{
if(request.AssumeStationaryCamera==request.UseCameraRotation)throw new ArgumentException("Choose either an explicitly stationary camera or the camera rotation followed during capture.");
cancellation.ThrowIfCancellationRequested();
var sourceBytes=File.ReadAllBytes(request.Motion);var source=MotionDocument.Parse(sourceBytes);
if(source.Space!=MotionSpace.CameraRelative||source.Bones.Count<22||!source.ModelVersion.Contains(GvhmrTemporalNetwork.CheckpointSha256,StringComparison.Ordinal))
throw new ArgumentException("Open the original camera-relative GVHMR reconstruction before refining it.");
var predictions=Path.Combine(Path.GetDirectoryName(Path.GetFullPath(request.Motion))!,"raw-predictions.json");
if(!File.Exists(predictions))throw new FileNotFoundException("Saved GVHMR predictions are missing. Open raw-body.hmotion from its original reconstruction folder.",predictions);
if(new FileInfo(predictions).Length>128*1024*1024)throw new InvalidDataException("Saved GVHMR predictions exceed the processing budget.");
var predictionBytes=File.ReadAllBytes(predictions);
var rawHash=Convert.ToHexString(SHA256.HashData(sourceBytes));var predictionHash=Convert.ToHexString(SHA256.HashData(predictionBytes));
var moving=request.UseCameraRotation;byte[]? rotationBytes=null;
if(moving)
{
var rotationPath=Path.Combine(Path.GetDirectoryName(Path.GetFullPath(request.Motion))!,CameraRotationFile);
if(!File.Exists(rotationPath))throw new FileNotFoundException("This capture has no followed camera rotation. Process the video again, or refine it as a stationary camera.",rotationPath);
rotationBytes=File.ReadAllBytes(rotationPath);predictionHash+="|"+Convert.ToHexString(SHA256.HashData(rotationBytes));
}
// The derived document embeds its original's location for reversible editing.
// Identical captures copied elsewhere must not restore an unrelated old path.
var key=Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes((moving?MovingVersion:Version)+"|"+Path.GetFullPath(request.Motion)+"|"+rawHash+"|"+predictionHash+"|"+SmplxSkeleton.NeutralSha256)));
var folder=Path.Combine(Path.GetFullPath(request.Output),key);Directory.CreateDirectory(folder);
using var jobLock=new FileStream(Path.Combine(folder,"job.lock"),FileMode.OpenOrCreate,FileAccess.ReadWrite,FileShare.None);
var destination=Path.Combine(folder,"contact-body.hmotion");
if(File.Exists(destination)&&File.Exists(Path.Combine(folder,"complete.json")))
{
var bytes=File.ReadAllBytes(destination);using var receipt=JsonDocument.Parse(File.ReadAllText(Path.Combine(folder,"complete.json")));
if(!receipt.RootElement.TryGetProperty("outputSha256",out var digest)||digest.GetString()!=Convert.ToHexString(SHA256.HashData(bytes)))
throw new InvalidDataException("The cached refinement changed. Choose a new output folder to rebuild it; the original capture is preserved.");
MotionDocument.Parse(bytes);progress?.Invoke("Reusing cached stationary-camera refinement");return destination;
}
var watch=Stopwatch.StartNew();progress?.Invoke("Reading saved body predictions; no neural inference");
var prediction=JsonSerializer.Deserialize<GvhmrTemporalNetwork.Output>(predictionBytes)??throw new InvalidDataException("Empty GVHMR prediction cache.");
if(prediction.Frames!=source.Frames.Count)throw new InvalidDataException("Saved predictions do not match the motion sample count.");
var pose=GvhmrDecoder.Decode(prediction.PredX,prediction.Frames);
var skeletonPath=Path.Combine(request.Models,"smplx/SMPLX_NEUTRAL.npz");
if(!File.Exists(skeletonPath))throw new FileNotFoundException("The SMPL-X neutral model used by body capture is missing. Restore SMPLX_NEUTRAL.npz in the configured models/smplx folder.",skeletonPath);
var skeleton=new SmplxSkeleton(skeletonPath,cancellation);
var rest=skeleton.RestPose(pose.Betas.AsSpan(0,10));
var cameraTranslation=source.Frames.Select(f=>Vector3.Transform(MotionDocument.V(f.Positions[0]),Quaternion.CreateFromAxisAngle(Vector3.UnitX,MathF.PI))-rest[0]).ToArray();
var stationary=Enumerable.Range(0,pose.Frames).SelectMany(_=>new[]{1f,0,0,0,1,0}).ToArray();
var followed=rotationBytes is null?null:JsonSerializer.Deserialize<CameraRotation>(rotationBytes)??throw new InvalidDataException("Empty camera rotation track.");
var rotation=followed?.AngularVelocity6d??stationary;
if(rotation.Length!=pose.Frames*6)throw new InvalidDataException("The camera rotation track does not match the motion sample count.");
if(followed is {RotationOnly:true})
{
// A camera turning about a fixed point: the pelvis seen from frame t, turned back by the
// accumulated rotation, is the pelvis in the first frame's camera axes, as if it had stood still.
var orientation=Quaternion.Identity;
for(var t=0;t<pose.Frames;t++)
{
cameraTranslation[t]=Vector3.Transform(cameraTranslation[t],Quaternion.Conjugate(orientation));
orientation=Quaternion.Normalize(GvhmrDecoder.Rotation6D(rotation.AsSpan(t*6,6))*orientation);
}
}
var root=GvhmrDecoder.WorldRoot(pose,rotation);
var raw=BodyMotionBuilder.WorldRelative(skeleton,pose,root,source,false);
progress?.Invoke("Correcting stationary-camera root and contact targets");
// Camera-space pelvis positions only anchor the root when the camera itself stood still.
var anchored=!moving||followed is {RotationOnly:true};
var correction=GvhmrContactProcessing.CorrectRoot(skeleton,pose,root,prediction.StaticConfidenceLogits,anchored?cameraTranslation:null,cancellation);
progress?.Invoke("Refining source limb contacts");
var refinedPose=pose with {BodyRotations=GvhmrLimbIk.Solve(skeleton,pose,correction.Root,correction.ContactTargets,cancellation)};
var refined=BodyMotionBuilder.WorldRelative(skeleton,refinedPose,correction.Root,source,true);
for(var channel=0;channel<6;channel++)
{
int c=channel;
refined.StationaryJoints.Add(new(){Bone=refined.Bones[GvhmrContactProcessing.ContactJoints[c]].Name,
Source="GVHMR static-joint head; uncalibrated contact probability, not visibility or 3D confidence",
Probability=Enumerable.Range(0,pose.Frames).Select(f=>1/(1+MathF.Exp(-prediction.StaticConfidenceLogits[f*6+c]))).ToArray()});
}
// Finger tracks hang beneath the wrists, unaffected by root or limb refinement, and arrive with the copied source.
BodyHandTracks.CarryFingerNotes(source,raw);BodyHandTracks.CarryFingerNotes(source,refined);
// Seated moments found by the capture carry over; the refinement rebuilds the stationary joints.
foreach(var seated in source.StationaryJoints.Where(s=>s.Source==SeatedDetection.Source))
foreach(var doc in new[]{raw,refined})if(doc.StationaryJoints.All(s=>s.Source!=SeatedDetection.Source))doc.StationaryJoints.Add(seated);
raw.OriginalReconstruction=new(){Path=Path.GetFullPath(request.Motion),Sha256=rawHash};
refined.OriginalReconstruction=new(){Path=Path.GetFullPath(request.Motion),Sha256=rawHash};
refined.ModelVersion+="; "+(moving?MovingVersion:Version);
refined.Diagnostics.Add("Original camera-relative reconstruction: "+Path.GetFullPath(request.Motion));
refined.Corrections.Add(new(){Type=moving?"GVHMR followed-camera-rotation contact refinement":"GVHMR stationary-camera contact refinement",Start=source.Frames[0].Time,End=source.Frames[^1].Time,Settings=new(){["stationaryCameraAssumption"]=moving?0:1,["ccdIterations"]=2}});
if(moving)
{
// The builder's notes describe the still-camera assumption, which was not made here.
refined.Diagnostics.RemoveAll(d=>d.Contains("stationary-camera",StringComparison.OrdinalIgnoreCase)&&!d.StartsWith("Original camera-relative",StringComparison.Ordinal));
refined.Diagnostics.Add("Two-iteration limb CCD applied to source limb contacts. Limb transforms affected by IK are labeled GeneratedIk.");
}
if(moving)refined.Diagnostics.Add(anchored
?"World-relative root from GVHMR's gravity-view rollout with the camera rotation followed from the background. The background showed little parallax, so the camera is treated as turning in place and camera-space pelvis positions, turned back by that rotation, anchor the root as for a still camera. A camera that also travelled would make travel distance wrong."
:"World-relative root from GVHMR's gravity-view rollout with the camera rotation followed from the background; static-joint root correction without a camera-space anchor, because background parallax shows the camera also travelled. Camera translation and scale are not recovered, so travel distance remains the model's estimate.");
cancellation.ThrowIfCancellationRequested();
void Write(string path,string json){File.WriteAllText(path+".partial",json);File.Move(path+".partial",path,true);}
var refinedJson=refined.ToJson();
Write(Path.Combine(folder,"raw-world.hmotion"),raw.ToJson());Write(destination,refinedJson);
Write(Path.Combine(folder,"complete.json"),JsonSerializer.Serialize(new{version=Version,source=Path.GetFullPath(request.Motion),rawHash,predictionHash,
stationaryCameraAssumption=!moving,frames=pose.Frames,seconds=watch.Elapsed.TotalSeconds,peakWorkerRamBytes=Process.GetCurrentProcess().PeakWorkingSet64,
neuralInference=false,metricScaleCalibrated=false,outputSha256=Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(refinedJson)))},MotionDocument.JsonOptions));
progress?.Invoke((moving?"Moving-camera":"Stationary-camera")+" refinement complete; review contacts before export");return destination;
}
}