InferenceWorker/WildHandsCrop.cs
using System.Runtime.InteropServices;
using HumanoidMocap.Inference;
using OpenCvSharp;

namespace HumanoidMocap.Worker;

/// <summary>WildHands demo preprocessing: its 840px square image preparation,
/// followed by the 224px dataset transform. Intrinsics and detector boxes follow
/// both transforms; the source footage remains unchanged.</summary>
public static class WildHandsCrop
{
    public const string ImplementationVersion="wildhands-demo840-v1";
    public sealed record Box(float Left,float Top,float Right,float Bottom);
    public sealed record Camera(float Fx,float Fy,float Cx,float Cy,bool Calibrated);
    public sealed record Prepared(float[] Image,float[] Right,float[] Left,float[] RightCenter,
        float[] RightCorners,float[] LeftCenter,float[] LeftCorners,Camera PatchCamera);
    public static Prepared Prepare(DecodedVideoFrame frame,Box? right,Box? left,Camera camera)
    {
        if(frame.Width<1||frame.Height<1||frame.Rgba.Length!=checked(frame.Width*frame.Height*4))throw new ArgumentException("Invalid video frame.");
        if(camera.Fx<=0||camera.Fy<=0||!new[]{camera.Fx,camera.Fy,camera.Cx,camera.Cy}.All(float.IsFinite))throw new ArgumentException("Invalid camera intrinsics.");
        using var rgba=new Mat(frame.Height,frame.Width,MatType.CV_8UC4);Marshal.Copy(frame.Rgba,0,rgba.Data,frame.Rgba.Length);
        using var rgb=new Mat();Cv2.CvtColor(rgba,rgb,ColorConversionCodes.RGBA2RGB);
        // demo.py prepares this image before dataset.py's Gaussian blur. Skipping
        // the first resize changes the blur and therefore the network input.
        const int size=840;var extent=Math.Max(frame.Width,frame.Height);var demoScale=size/(float)extent;
        float DemoX(float x)=>(x-frame.Width*.5f)*demoScale+size*.5f;
        float DemoY(float y)=>(y-frame.Height*.5f)*demoScale+size*.5f;
        Box? ResizeBox(Box? box)=>box is null?null:new(DemoX(box.Left),DemoY(box.Top),DemoX(box.Right),DemoY(box.Bottom));
        right=ResizeBox(right);left=ResizeBox(left);
        camera=new(camera.Fx*demoScale,camera.Fy*demoScale,DemoX(camera.Cx),DemoY(camera.Cy),camera.Calibrated);
        using var demo=Crop(rgb,frame.Width*.5f,frame.Height*.5f,extent,size);
        using var blurred=new Mat();Cv2.GaussianBlur(demo,blurred,new Size(5,5),8);
        var scale=224f/size;
        using var square=Crop(blurred,size*.5f,size*.5f,size);
        using var global=new Mat();square.ConvertTo(global,MatType.CV_32FC3,1d/255);
        var patchCamera=new Camera(camera.Fx*scale,camera.Fy*scale,
            (camera.Cx-size*.5f)*scale+112,(camera.Cy-size*.5f)*scale+112,camera.Calibrated);
        (float[] Image,float[] Center,float[] Corners) Hand(Box? box)
        {
            float cx=112,cy=112,extent=224;int x0=0,y0=0,x1=223,y1=223;
            if(box is not null)
            {
                if(!new[]{box.Left,box.Top,box.Right,box.Bottom}.All(float.IsFinite)||box.Right<=box.Left||box.Bottom<=box.Top)
                    throw new ArgumentException("Invalid hand crop.");
                int X(float x)=>Math.Clamp((int)(scale*(x-size*.5f)+112)+1,0,223);
                int Y(float y)=>Math.Clamp((int)(scale*(y-size*.5f)+112)+1,0,223);
                x0=X(box.Left);y0=Y(box.Top);x1=X(box.Right);y1=Y(box.Bottom);
                cx=(x0+x1)/2;cy=(y0+y1)/2;extent=Math.Max(x1-x0,y1-y0)*1.75f;
                if(extent<1)throw new ArgumentException("Hand crop collapses outside the image.");
                var half=MathF.Floor(extent/2);
                x0=Math.Clamp((int)(cx-half),0,223);x1=Math.Clamp((int)(cx+half),0,223);
                y0=Math.Clamp((int)(cy-half),0,223);y1=Math.Clamp((int)(cy+half),0,223);
            }
            using var crop=Crop(global,cx,cy,extent);
            float AngleX(float x)=>MathF.Atan2(x-patchCamera.Cx,patchCamera.Fx);
            float AngleY(float y)=>MathF.Atan2(y-patchCamera.Cy,patchCamera.Fy);
            return(Normalize(crop),new[]{AngleX((x0+x1)/2f),AngleY((y0+y1)/2f)},
                new[]{AngleX(x0),AngleY(y0),AngleX(x0),AngleY(y1),AngleX(x1),AngleY(y0),AngleX(x1),AngleY(y1)});
        }
        var r=Hand(right);var l=Hand(left);
        return new(Normalize(global),r.Image,l.Image,r.Center,r.Corners,l.Center,l.Corners,patchCamera);
    }
    static Mat Crop(Mat image,float cx,float cy,float size,int resolution=224)
    {
        using var affine=Cv2.GetAffineTransform(new[]{new Point2f(cx,cy),new Point2f(cx,cy+size/2),new Point2f(cx+size/2,cy)},
            new[]{new Point2f(resolution*.5f,resolution*.5f),new Point2f(resolution*.5f,resolution),new Point2f(resolution,resolution*.5f)});
        using var affineFloat=new Mat();affine.ConvertTo(affineFloat,MatType.CV_32FC1);
        var result=new Mat();
        try{Cv2.WarpAffine(image,result,affineFloat,new Size(resolution,resolution),InterpolationFlags.Cubic,BorderTypes.Constant,Scalar.Black);return result;}
        catch{result.Dispose();throw;}
    }
    static float[] Normalize(Mat image)
    {
        var rgb=new float[224*224*3];Marshal.Copy(image.Data,rgb,0,rgb.Length);
        float[] mean={.485f,.456f,.406f},std={.229f,.224f,.225f};var result=new float[rgb.Length];
        for(var c=0;c<3;c++)for(var pixel=0;pixel<224*224;pixel++)result[c*224*224+pixel]=(Math.Clamp(rgb[pixel*3+c],0,1)-mean[c])/std[c];
        return result;
    }
}