Code/HumanoidMocap/Core/Motion/PropContactSuggestions.cs

Static utility that suggests contact intervals between hands and imported prop surfaces from a MotionDocument. It scans frames, computes palm position and finger curl, matches against triangulated prop surfaces sampled from PropContactMotion, and returns suggested ContactInterval entries with counts of ambiguous and usable samples.

File Access
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Threading;
using HumanoidMocap.Core.Mapping;
using HumanoidMocap.Core.Maths;

namespace HumanoidMocap.Core.Motion;
using Vector3 = System.Numerics.Vector3;

/// <summary>Reviewable grip intervals from observed hand shape and imported rigid surfaces.
/// Geometry measures are heuristics, never backend confidence or object tracking.</summary>
public static class PropContactSuggestions
{
    sealed record Surface(string Prop,string Bone,int Index,ContactSurfaceIndex Search);
    public sealed record Result(List<ContactInterval> Contacts,int AmbiguousSamples,int UsableSamples);
    public static Result Suggest(MotionDocument motion,CancellationToken cancellation=default)
    {
        cancellation.ThrowIfCancellationRequested();
        motion.Validate();
        if(!HandCaptureRetargeter.Supports(motion)||motion.Space!=MotionSpace.CameraRelative)
            throw new ArgumentException("Contact suggestions currently require camera-relative hand capture.");
        if(motion.ModelVersion.Contains("hand-forest-v2-parent-swing",StringComparison.Ordinal))
            throw new InvalidOperationException("Reprocess this older MediaPipe capture before suggesting contacts. The worker reuses cached observations and preserves the existing motion file.");
        var surfaces=motion.Objects.SelectMany(p=>p.Surfaces.GroupBy(s=>s.Bone).Select(g=>{
            var v=new List<float[]>();var t=new List<int>();foreach(var s in g){var offset=v.Count;v.AddRange(s.Vertices);t.AddRange(s.Triangles.Select(i=>i+offset));}
            return new Surface(p.Id,g.Key,p.Bones.FindIndex(b=>b.Name==g.Key),new(new(){Vertices=v.ToArray(),Triangles=t.ToArray()}));
        })).ToArray();
        if(surfaces.Length==0)throw new ArgumentException("No rigid contact surfaces were imported. Include a triangulated mesh with rigid bone weights in the prop FBX.");
        var tracks=new PropContactMotion(motion);var settings=new ContactSettings();var output=new List<ContactInterval>();
        var ambiguous=0;var usable=0;long budget=20_000_000;
        foreach(var left in new[]{true,false})
        {
            int Bone(string role)=>motion.Bones.FindIndex(b=>b.Role==Enum.Parse<BoneRole>(role+(left?"L":"R")));
            var hand=Bone("Hand");if(hand<0)continue;
            var fingers=new[]{"Index","Middle","Ring","Pinky"}.Select(n=>new[]{Bone(n+"Prox"),Bone(n+"Mid"),Bone(n+"Dist")}).Where(v=>v.All(i=>i>=0)).ToArray();
            if(fingers.Length<3)continue;
            var winners=new int[motion.Frames.Count];Array.Fill(winners,-1);var samples=new ContactSample[motion.Frames.Count];
            var world=new XForm[motion.Bones.Count];var valid=new bool[motion.Bones.Count];
            for(var f=0;f<motion.Frames.Count;f++)
            {
                cancellation.ThrowIfCancellationRequested();var frame=motion.Frames[f];
                if(frame.Evidence[hand]!=JointEvidence.Reconstructed)continue;
                for(var b=0;b<world.Length;b++)
                {var parent=motion.Bones[b].Parent;var local=new XForm(MotionDocument.V(frame.Positions[b]),MotionDocument.Q(frame.Rotations[b]));world[b]=parent<0?local:XForm.Compose(world[parent],local);
                    // A canonical metacarpal can bridge the observed wrist and finger.
                    // This is authored anatomy, never evidence of an observed finger.
                    var authoredMeta=frame.Evidence[b]==JointEvidence.Authored&&motion.Bones[b].Role is {} role&&role.ToString().Contains("Meta",StringComparison.Ordinal);
                    valid[b]=(frame.Evidence[b]==JointEvidence.Reconstructed||authoredMeta)&&(parent<0||valid[parent]);}
                if(!valid[hand])continue;
                var knuckles=Vector3.Zero;float curl=0;int observed=0;
                foreach(var finger in fingers)
                {
                    if(finger.Any(b=>!valid[b]||frame.Evidence[b]!=JointEvidence.Reconstructed))continue;
                    var a=world[finger[1]].Pos-world[finger[0]].Pos;var b=world[finger[2]].Pos-world[finger[1]].Pos;
                    if(a.LengthSquared()<1e-10f||b.LengthSquared()<1e-10f)continue;
                    curl+=(1-Math.Clamp(Vector3.Dot(Vector3.Normalize(a),Vector3.Normalize(b)),-1,1))*.5f;
                    knuckles+=world[finger[0]].Pos;observed++;
                }
                if(observed<3)continue;
                var palm=(world[hand].Pos+knuckles/observed)*.5f;curl/=observed;
                float best=settings.ExitDistance,second=float.PositiveInfinity;int winner=-1;Vector3 closest=default;XForm objectPose=default;
                var poses=new Dictionary<string,(XForm[] World,bool[] Valid)>();
                for(var s=0;s<surfaces.Length;s++)
                {
                    if(--budget<0)throw new InvalidOperationException("Contact search exceeded its work budget. Use a shorter clip or simpler prop geometry.");
                    var surface=surfaces[s];
                    if(!poses.TryGetValue(surface.Prop,out var pose))
                    {
                        if(!tracks.TrySample(surface.Prop,frame.Time,out var w,out var available))continue;
                        poses.Add(surface.Prop,pose=(w,available));
                    }
                    if(!pose.Valid[surface.Index])continue;
                    var obj=pose.World[surface.Index];var local=XForm.Compose(obj.Inverse(),new(palm,Quaternion.Identity)).Pos;
                    if(!surface.Search.TryClosest(local,settings.ExitDistance,ref budget,out var near))continue;
                    var distance=Vector3.Distance(local,near);
                    if(distance<best){if(winner>=0)second=best;best=distance;winner=s;closest=XForm.Compose(obj,new(near,Quaternion.Identity)).Pos;objectPose=obj;}
                    else second=Math.Min(second,distance);
                }
                if(winner<0)continue;
                // Two surfaces at almost the same distance are insufficient evidence
                // to choose an object or articulated part. Leave them for manual review.
                if(second-best<.005f){ambiguous++;continue;}
                winners[f]=winner;samples[f]=new(frame.Time,palm,objectPose,closest,curl,true,world[hand].Pos);usable++;
            }
            foreach(var index in winners.Where(i=>i>=0).Distinct())
            {
                cancellation.ThrowIfCancellationRequested();
                var stream=Enumerable.Range(0,winners.Length).Select(f=>winners[f]==index?samples[f]:new ContactSample(motion.Frames[f].Time,Vector3.Zero,XForm.Identity,Vector3.Zero,0,false)).ToArray();
                var surface=surfaces[index];var suggested=ContactSolver.Suggest(stream,motion.Bones[hand].Name,surface.Prop,settings);
                foreach(var contact in suggested)
                {
                    contact.ObjectBone=surface.Bone;
                    contact.Reason="Imported rigid surface proximity, observed finger bend, relative motion and persistence. Wrist anchor preserves the initial wrist offset. Uncertain heuristic; review against the video.";
                    // Existing reviewed or dismissed intervals are user decisions.
                    if(!motion.Contacts.Any(c=>c.Bone==contact.Bone&&c.End>=contact.Start&&c.Start<=contact.End))output.Add(contact);
                }
            }
        }
        return new(output.OrderBy(c=>c.Start).ThenBy(c=>c.Bone).ToList(),ambiguous,usable);
    }
}