Editor/HumanoidMocap/Inference/ManagedHands.cs
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Numerics;
using System.Threading;
namespace HumanoidMocap.Inference;
using Vector2 = System.Numerics.Vector2;
using Vector3 = System.Numerics.Vector3;
/// <summary>Experimental managed port of MediaPipe's palm + landmark inference graphs.
/// Image landmarks are pixels. World landmarks are hand-relative metres, not world tracking.</summary>
public sealed class ManagedHands
{
public const string ImplementationVersion="managed-hands-v9-track-conflicts";
/// <summary>Runs one TFLite model: NHWC input in, its outputs out.</summary>
public delegate float[][] ModelRunner(float[] input,CancellationToken token);
/// <summary>Builds the runner for each model. The managed interpreter by default; the capture worker
/// substitutes ONNX Runtime, which runs the same operators with optimised native kernels.</summary>
public static Func<LiteModel,ModelRunner> RunnerFactory { get; set; }=model=>new LiteInterpreter(model).Run;
readonly ModelRunner palms,hands;
readonly List<Vector2> anchors=new();
public ManagedHands(byte[] task)
{
using var zip=new ZipArchive(new MemoryStream(task),ZipArchiveMode.Read);
ModelRunner Load(string name)
{
using var input=(zip.GetEntry(name)??throw new FormatException("Missing "+name)).Open();
using var memory=new MemoryStream();input.CopyTo(memory);return RunnerFactory(new LiteModel(memory.ToArray()));
}
palms=Load("hand_detector.tflite");hands=Load("hand_landmarks_detector.tflite");
foreach(var (size,repeats) in new[]{(24,2),(12,6)})
for(var y=0;y<size;y++)for(var x=0;x<size;x++)for(var a=0;a<repeats;a++)anchors.Add(new((x+.5f)/size,(y+.5f)/size));
}
public List<HandObservation> Detect(byte[] rgba,int width,int height,CancellationToken token=default)
=>Observe(rgba,width,height,PalmRects(rgba,width,height,token),token);
sealed record HandRect(float X,float Y,float Size,float Angle,bool Tracked=false);
List<HandRect> PalmRects(byte[] rgba,int width,int height,CancellationToken token)
{
if(rgba.Length!=width*height*4)throw new ArgumentException("RGBA buffer size mismatch.");
var longest=Math.Max(width,height);var input=Crop(rgba,width,height,192,width/2f,height/2f,longest,0);
var output=palms(input,token);var candidates=new List<PalmDetection>();
for(var i=0;i<anchors.Count;i++)
{
var score=1/(1+MathF.Exp(-Math.Clamp(output[1][i],-100,100)));if(score<.5f)continue;
var offset=i*18;
Vector2 Point(int j)=>new((output[0][offset+j]/192+anchors[i].X-.5f)*longest+width/2f,
(output[0][offset+j+1]/192+anchors[i].Y-.5f)*longest+height/2f);
var center=Point(0);var points=Enumerable.Range(0,7).Select(j=>Point(4+2*j)).ToArray();
candidates.Add(new(score,center.X,center.Y,output[0][offset+2]/192*longest,output[0][offset+3]/192*longest,points));
}
var selected=PalmDetectionFilter.Merge(candidates);
var result=new List<HandRect>();
foreach(var palm in selected)
{
var direction=palm.Points[2]-palm.Points[0];
var angle=MathF.PI/2+MathF.Atan2(direction.Y,direction.X);
var cos=MathF.Cos(angle);var sin=MathF.Sin(angle);
var cx=palm.X+.5f*palm.Height*sin;var cy=palm.Y-.5f*palm.Height*cos;
var size=Math.Max(palm.Width,palm.Height)*2.6f;
if(size<2)continue;
result.Add(new(cx,cy,size,angle));
}
return result;
}
/// <summary>Prefer prior landmark ROIs as in MediaPipe's video graph. Detect
/// new palms only while fewer than two tracks remain. Every output still
/// requires fresh landmark inference and presence; lost tracks are not poses.</summary>
public List<HandObservation> DetectTracked(byte[] rgba,int width,int height,IReadOnlyList<HandObservation> previous,CancellationToken token=default)
{
if(rgba.Length!=width*height*4)throw new ArgumentException("RGBA buffer size mismatch.");
var regions=new List<HandRect>();
foreach(var prior in previous.Take(2))
{
if(prior.ImageLandmarks.Length!=21)continue;
if(RegionFromLandmarks(prior.ImageLandmarks,width,height) is { } region)regions.Add(region);
}
if(regions.Count<2)
foreach(var candidate in PalmRects(rgba,width,height,token))
{
// Upstream HandAssociation uses axis-aligned ROI overlap and
// prioritizes the prior regions. Redetection follows loss on
// the next frame, without replacing successful crops each time.
if(regions.Any(r=>RegionIou(r,candidate)>.5f))continue;
regions.Add(candidate);if(regions.Count==2)break;
}
var current=Observe(rgba,width,height,regions,token);
current=HandIdentity.PreserveSeparatedTracks(current,previous);
current=HandIdentity.ResolveConflicts(current,previous);
return current.GroupBy(h=>h.Side).Select(g=>g.OrderByDescending(h=>h.Presence).First()).ToList();
}
/// <summary>Landmark inference in the region implied by 21 image landmarks from any source,
/// for example another model's projected joints. Returns the landmark model's own presence
/// score and its observation even below the usual 0.5 acceptance, so a caller can weigh it.</summary>
public (float Presence,HandObservation? Observation) Probe(byte[] rgba,int width,int height,Vector3[] imageLandmarks,CancellationToken token=default)
{
if(rgba.Length!=width*height*4)throw new ArgumentException("RGBA buffer size mismatch.");
if(imageLandmarks.Length!=21||RegionFromLandmarks(imageLandmarks,width,height) is not { } r)return(0,null);
var observation=Landmarks(rgba,width,height,r.X,r.Y,r.Size,r.Angle,token,0);
return(observation?.Presence??0,observation is null?null:observation with{Tracked=true});
}
static HandRect? RegionFromLandmarks(Vector3[] points,int width,int height)
{
var direction=(points[5]+points[13]+2*points[9])*.25f-points[0];
var angle=MathF.PI/2+MathF.Atan2(direction.Y,direction.X);
var cos=MathF.Cos(angle);var sin=MathF.Sin(angle);
// MediaPipe's HandLandmarksToRect partial set and ROI graph: the
// palm/proximal landmarks, square-long scale 2 and local y shift -0.1.
var subset=new[]{0,1,2,3,5,6,9,10,13,14,17,18};
var rotated=subset.Select(i=>new Vector2(cos*points[i].X+sin*points[i].Y,-sin*points[i].X+cos*points[i].Y)).ToArray();
var min=new Vector2(rotated.Min(p=>p.X),rotated.Min(p=>p.Y));
var max=new Vector2(rotated.Max(p=>p.X),rotated.Max(p=>p.Y));
var center=(min+max)*.5f;center.Y-=(max.Y-min.Y)*.1f;
var size=Math.Max(max.X-min.X,max.Y-min.Y)*2;
if(!float.IsFinite(size)||size<2||size>Math.Max(width,height)*2)return null;
return new(cos*center.X-sin*center.Y,sin*center.X+cos*center.Y,size,angle,true);
}
List<HandObservation> Observe(byte[] rgba,int width,int height,IEnumerable<HandRect> regions,CancellationToken token)
{
var result=new List<HandObservation>();
foreach(var r in regions)
{
var observation=Landmarks(rgba,width,height,r.X,r.Y,r.Size,r.Angle,token);
if(observation is not null)result.Add(observation with{Tracked=r.Tracked});
}
return result;
}
static float RegionIou(HandRect a,HandRect b)
{
var width=Math.Max(0,Math.Min(a.X+a.Size/2,b.X+b.Size/2)-Math.Max(a.X-a.Size/2,b.X-b.Size/2));
var height=Math.Max(0,Math.Min(a.Y+a.Size/2,b.Y+b.Size/2)-Math.Max(a.Y-a.Size/2,b.Y-b.Size/2));
var intersection=width*height;return intersection/(a.Size*a.Size+b.Size*b.Size-intersection);
}
HandObservation? Landmarks(byte[] rgba,int width,int height,float cx,float cy,float size,float angle,CancellationToken token,float minimumPresence=.5f)
{
var cos=MathF.Cos(angle);var sin=MathF.Sin(angle);
var predictions=hands(Crop(rgba,width,height,224,cx,cy,size,angle,replicateBorder:true),token);
if(predictions[1][0]<minimumPresence)return null;
var screen=new Vector3[21];var world=new Vector3[21];
for(var i=0;i<21;i++)
{
var x=(predictions[0][i*3]/224-.5f)*size;var y=(predictions[0][i*3+1]/224-.5f)*size;
screen[i]=new(cx+cos*x-sin*y,cy+sin*x+cos*y,predictions[0][i*3+2]/224/.4f*size);
x=predictions[3][i*3];y=predictions[3][i*3+1];world[i]=new(cos*x-sin*y,sin*x+cos*y,predictions[3][i*3+2]);
}
// Preserve the task model's handedness label; mirrored footage can be corrected
// explicitly with Swap hands. Verified against the visible stirring hand sample.
var right=predictions[2][0];return new(right>.5f?"R":"L",predictions[1][0],Math.Max(right,1-right),screen,world);
}
static float[] Crop(byte[] rgba,int width,int height,int size,float cx,float cy,float span,float angle,bool replicateBorder=false)
{
var result=new float[size*size*3];var cos=Math.Cos(angle);var sin=Math.Sin(angle);
for(var y=0;y<size;y++)for(var x=0;x<size;x++)
{
// MediaPipe's CPU converter maps ROI corners to [0,size] and samples
// at integer destination pixels. OpenCV INTER_LINEAR quantizes the
// fractional coordinates to 1/32, then rounds interpolated RGB bytes.
var dx=((double)x/size-.5)*span;var dy=((double)y/size-.5)*span;
var px=(int)Math.Round((cx+cos*dx-sin*dy)*32);var py=(int)Math.Round((cy+sin*dx+cos*dy)*32);
var ix=px>>5;var iy=py>>5;var fx=px&31;var fy=py&31;
for(var c=0;c<3;c++)
{
int Sample(int sx,int sy)
{
if(replicateBorder){sx=Math.Clamp(sx,0,width-1);sy=Math.Clamp(sy,0,height-1);}
return sx<0||sy<0||sx>=width||sy>=height?0:rgba[(sy*width+sx)*4+c];
}
var value=Sample(ix,iy)*(32-fx)*(32-fy)+Sample(ix+1,iy)*fx*(32-fy)
+Sample(ix,iy+1)*(32-fx)*fy+Sample(ix+1,iy+1)*fx*fy;
result[(y*size+x)*3+c]=((value+512)>>10)/255f;
}
}
return result;
}
}