InferenceWorker/PersonCropTrack.cs
using HumanoidMocap.Inference;

namespace HumanoidMocap.Worker;

/// <summary>Conservative single-subject association in image coordinates. Long loss or
/// ambiguous subjects require a shorter range or an explicit manual crop.</summary>
public static class PersonCropTrack
{
    public sealed record Frame(double Time,PersonDetector.Detection[] Detections);
    public sealed record Sample(GvhmrDecoder.Box Crop,string Evidence,float? DetectorScore);
    /// <summary>The detection that continues the followed subject, or the single prominent
    /// subject while none is followed. Null when absent or ambiguous.</summary>
    public static PersonDetector.Detection? Select(GvhmrDecoder.Box? previous,IReadOnlyList<PersonDetector.Detection> detections)
    {
        var candidates=detections.Where(d=>d.Score>=.5f&&float.IsFinite(d.Score)&&
            d.Landmarks.Length==8&&d.Landmarks.All(float.IsFinite)&&d.Crop.Size>=16&&float.IsFinite(d.Crop.Size)).ToArray();
        if(previous is not { } p)
        {
            // Prefer the prominent foreground subject; do not pick arbitrarily
            // when two candidates have comparable apparent size and evidence.
            var prominent=candidates.OrderByDescending(d=>d.Crop.Size*d.Score).ToArray();
            return prominent.Length>0&&(prominent.Length==1||prominent[1].Crop.Size*prominent[1].Score<prominent[0].Crop.Size*prominent[0].Score*.8f)?prominent[0]:null;
        }
        var ranked=candidates.Select(d=>new{Detection=d,Distance=Distance(d.Crop,p)/p.Size,Scale=d.Crop.Size/p.Size})
            .Where(d=>d.Distance<.65f&&d.Scale>.5f&&d.Scale<2)
            .Select(d=>new{d.Detection,Cost=d.Distance+Math.Abs(MathF.Log(d.Scale))*.4f+(1-d.Detection.Score)*.1f})
            .OrderBy(d=>d.Cost).ToArray();
        return ranked.Length>0&&(ranked.Length==1||ranked[1].Cost-ranked[0].Cost>.15f)?ranked[0].Detection:null;
    }
    /// <summary>Square model crop around confidently located COCO-17 joints (x, y, score),
    /// or null when fewer than six are confident. The joint box excludes the top of the
    /// head, hands and feet, so it is padded more than a detector's person box would be.</summary>
    public static GvhmrDecoder.Box? FromJoints(float[] joints)
    {
        if(joints.Length!=51)throw new ArgumentException("Expected COCO-17 joints.");
        float left=float.MaxValue,top=float.MaxValue,right=float.MinValue,bottom=float.MinValue;var confident=0;
        for(var j=0;j<17;j++)
        {
            if(!(joints[j*3+2]>=.5f)||!float.IsFinite(joints[j*3]+joints[j*3+1]))continue;
            confident++;left=Math.Min(left,joints[j*3]);right=Math.Max(right,joints[j*3]);top=Math.Min(top,joints[j*3+1]);bottom=Math.Max(bottom,joints[j*3+1]);
        }
        if(confident<6)return null;
        var size=Math.Max(right-left,bottom-top)*1.4f;
        return size>=16?new((left+right)/2,(top+bottom)/2,size):null;
    }
    /// <summary>Whether these joints show a body that can be followed. <paramref name="previousSpan"/> is the
    /// <see cref="Span"/> of the subject's last sighting, or null on first acquisition.
    /// As a performer walked out of the picture the pose model kept answering: first with joints piled
    /// along the border they left by, then with a confident "body" 30 pixels tall on a shelf the detector
    /// had offered, which kept a capture going for a second after the person was gone. A followable body
    /// has six confident joints clear of the picture's border, some of its torso among them, and has not
    /// suddenly shrunk to a fraction of its size a moment ago.</summary>
    public static bool Followable(float[] joints,float? previousSpan,int width,int height)
        =>FromJoints(joints) is not null&&Span(joints,width,height) is float span&&(previousSpan is not float known||span>=known*MinimumSpanKept);
    /// <summary>Larger side of the box around confident joints clear of the border, or null when fewer than six
    /// are, or none of them is a shoulder or hip.</summary>
    public static float? Span(float[] joints,int width,int height)
    {
        float marginX=width*BorderFraction,marginY=height*BorderFraction,left=float.MaxValue,top=float.MaxValue,right=float.MinValue,bottom=float.MinValue;
        var inside=0;var torso=false;
        for(var j=0;j<17;j++)
        {
            float x=joints[j*3],y=joints[j*3+1];
            if(!(joints[j*3+2]>=.5f)||x<marginX||x>width-marginX||y<marginY||y>height-marginY)continue;
            inside++;torso|=j is 5 or 6 or 11 or 12;left=Math.Min(left,x);right=Math.Max(right,x);top=Math.Min(top,y);bottom=Math.Max(bottom,y);
        }
        return inside>=6&&torso?Math.Max(right-left,bottom-top):null;
    }
    public const float BorderFraction=.03f,MinimumSpanKept=.4f;
    public static int ConfidentJoints(float[] joints)=>Enumerable.Range(0,17).Count(j=>joints[j*3+2]>=.5f);
    /// <summary>Next crop while following one subject. A box around partly visible joints is
    /// smaller than the body, and a smaller crop hides more joints on the next frame; on the
    /// kata sample that collapsed a 292-pixel crop to 54 pixels within five frames. Size may
    /// therefore shrink by at most 8% per frame, only while at least twelve joints are
    /// confident, and grow by at most 35%.</summary>
    public static GvhmrDecoder.Box Continue(GvhmrDecoder.Box? previous,GvhmrDecoder.Box fromJoints,int confidentJoints)
    {
        if(previous is not { } p)return fromJoints;
        var smallest=confidentJoints>=12?p.Size*.92f:p.Size;
        return fromJoints with{Size=Math.Clamp(fromJoints.Size,smallest,p.Size*1.35f)};
    }
    public static Sample[] Build(IReadOnlyList<Frame> frames)
    {
        if(frames.Count==0)throw new ArgumentException("No frames for person tracking.");
        var result=new Sample[frames.Count];GvhmrDecoder.Box? previous=null;var lastObserved=-1;
        for(var i=0;i<frames.Count;i++)
        {
            if(!double.IsFinite(frames[i].Time)||(i>0&&frames[i].Time<=frames[i-1].Time))throw new ArgumentException("Person-track timestamps must increase.");
            var selected=Select(previous,frames[i].Detections);
            if(selected is null)
            {
                if(lastObserved<0||frames[i].Time-frames[lastObserved].Time>.25)
                    throw new InvalidDataException($"Cannot follow one person at {frames[i].Time:F2}s. Use a shorter range with one clearly visible subject or supply PersonCrop in a manual worker job.");
                continue;
            }
            var crop=selected.Crop;result[i]=new(crop,"detected",selected.Score);
            if(lastObserved>=0&&i>lastObserved+1)
            {
                if(frames[i].Time-frames[lastObserved].Time>.25)throw new InvalidDataException("Person tracking gap exceeds 0.25 seconds. Use a shorter range or a manual crop.");
                for(var missing=lastObserved+1;missing<i;missing++)
                {
                    var amount=(float)((frames[missing].Time-frames[lastObserved].Time)/(frames[i].Time-frames[lastObserved].Time));
                    var a=previous!.Value;
                    result[missing]=new(new(a.CenterX+(crop.CenterX-a.CenterX)*amount,a.CenterY+(crop.CenterY-a.CenterY)*amount,a.Size+(crop.Size-a.Size)*amount),"interpolated crop",null);
                }
            }
            previous=crop;lastObserved=i;
        }
        if(lastObserved!=frames.Count-1)throw new InvalidDataException("Person tracking is lost at the end. Trim the range or supply a manual crop.");
        return result;
    }
    static float Distance(GvhmrDecoder.Box a,GvhmrDecoder.Box b)=>MathF.Sqrt((a.CenterX-b.CenterX)*(a.CenterX-b.CenterX)+(a.CenterY-b.CenterY)*(a.CenterY-b.CenterY));

    /// <summary>GVHMR's tracker applies a centered five-frame average twice to its
    /// bounding boxes before neural reconstruction. This filters image crops only;
    /// original detections, timestamps and reconstructed motion remain separate.</summary>
    public static Sample[] Stabilize(Sample[] track)
    {
        var result=track.ToArray();
        for(var pass=0;pass<2;pass++)
        {
            var source=result;result=new Sample[source.Length];
            for(var i=0;i<result.Length;i++)
            {
                float x=0,y=0,size=0;
                for(var offset=-2;offset<=2;offset++)
                {
                    var box=source[Math.Clamp(i+offset,0,source.Length-1)].Crop;
                    x+=box.CenterX/5;y+=box.CenterY/5;size+=box.Size/5;
                }
                result[i]=source[i] with{Crop=new(x,y,size)};
            }
        }
        return result;
    }
}