dev/sam_frame_checks.cs
using System.Diagnostics;
using System.Text.Json;
using HumanoidMocapX.EditorTools;
using HumanoidMocapX.Inference;
using Sandbox;

namespace HumanoidMocapX.Checks;

/// <summary>Runs released YOLOX, SAM DINOv3, body decoder and MHR on a real frame.
/// This is an isolated body-pass diagnostic, not full video or hand refinement.</summary>
public static class SamFrameChecks
{
    /// <summary>Writes crop, keypoint and feature evidence to a JSON file. Requires
    /// a visible person in an RGBA8 frame and installed, verified model files.</summary>
    public static async Task<object> Run(string samPath, string mhrPath, string detectorPath, string rgbaPath,
        int width, int height, string resultPath, CancellationToken cancellation)
    {
        var pixels = await File.ReadAllBytesAsync(rgbaPath, cancellation);
        var box = await Detect(pixels, width, height, detectorPath, cancellation);
        var shape = DinoShape.Sam(); var crop = new SamCrop(box.Left, box.Top, box.Right, box.Bottom, shape);
        var focal = (float)Math.Sqrt((double)width * width + (double)height * height);
        var intrinsics = new float[] { focal, 0, width / 2f, 0, focal, height / 2f, 0, 0, 1 }; var affine = new float[6]; crop.CopyAffine(affine);
        var rgb = new float[shape.ImageElements]; var rayEncoder = new SamRays(shape); var rays = new float[rayEncoder.Elements];
        await Task.Run(() => { crop.Prepare(pixels, width, height, rgb, cancellation); rayEncoder.Prepare(intrinsics, affine, rays, cancellation); }, cancellation);
        var context = new float[shape.Patches * shape.Channels]; var positions = new float[context.Length];
        var imageTimes = await Encode(samPath, shape, rgb, rays, context, positions, cancellation);
        var model = await Task.Run(() => MhrCheckpoint.LoadBody(mhrPath, cancellation), cancellation);
        var pose = await Task.Run(() => SamCheckpoint.LoadPoseHead(samPath, model, false, cancellation), cancellation);
        var decoderShape = new SamDecoderShape(SamInitialTokens.Count, shape.Patches);
        var camera = await Task.Run(() => new SamPredictionHead(VisionCheckpoint.LoadSamHead(samPath, false, cancellation), false, true), cancellation);
        var keypoints = await Task.Run(() => new SamKeypointFeedback(VisionCheckpoint.LoadSamFeedback(samPath, false, cancellation), decoderShape, 32, 32), cancellation);
        var projection = new SamProjection(intrinsics, affine, crop.CenterX, crop.CenterY, crop.Size, shape);
        var feedback = new SamDecoderFeedback(decoderShape, pose, camera, keypoints); feedback.Prepare(projection, context);
        var tokens = new float[decoderShape.TokenElements]; var tokenPositions = new float[tokens.Length]; var decoded = new float[tokens.Length];
        await Task.Run(() => new SamInitialTokens(VisionCheckpoint.LoadSamInitialTokens(samPath, false, cancellation))
            .Run(projection, tokens, tokenPositions, cancellation), cancellation);
        var decoderWeights = await Task.Run(() => VisionCheckpoint.LoadSamDecoder(samPath, false, cancellation), cancellation);
        await GameTask.MainThread(); var decoder = new SamGpuDecoder(decoderWeights, decoderShape); var timer = Stopwatch.StartNew();
        try
        {
            await decoder.RunAsync(tokens, context, tokenPositions, positions, decoded, feedback.Update, cancellation);
            await Task.Run(() => feedback.Finish(decoded, cancellation), cancellation);
        }
        finally { await GameTask.MainThread(); decoder.Dispose(); }
        var decodeMilliseconds = timer.Elapsed.TotalMilliseconds;
        var cropped = new float[140]; var depths = new float[70]; var translation = new float[3]; var imagePoints = new float[140];
        projection.Run(pose.Keypoints, feedback.CameraParameters, cropped, depths, translation);
        for (var i = 0; i < 70; i++)
        {
            imagePoints[i * 2] = ((cropped[i * 2] + .5f) * shape.Width - affine[2]) / affine[0];
            imagePoints[i * 2 + 1] = ((cropped[i * 2 + 1] + .5f) * shape.Height - affine[5]) / affine[4];
        }
        var result = new
        {
            BodyPassOnly = true,
            Width = width,
            Height = height,
            Box = box,
            Crop = new { crop.CenterX, crop.CenterY, crop.Size },
            ImageMilliseconds = imageTimes,
            DecodeMilliseconds = decodeMilliseconds,
            ImageKeypoints = imagePoints,
            KeypointsMetres = pose.Keypoints.ToArray(),
            Depths = depths,
            Translation = translation,
            BodyFeature = decoded.AsSpan(0, decoderShape.Width).ToArray(),
            ModelParameters = pose.Model.ToArray()
        };
        await File.WriteAllTextAsync(resultPath, JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }), cancellation);
        return new
        {
            Passed = true,
            BodyPassOnly = true,
            Box = box,
            Keypoints = 70,
            BodyFeatureChannels = decoderShape.Width,
            ImageMilliseconds = imageTimes,
            DecodeMilliseconds = decodeMilliseconds,
            ResultPath = resultPath
        };
    }

    static async Task<YoloBox> Detect(byte[] pixels, int width, int height, string path, CancellationToken cancellation)
    {
        var network = new YoloNetwork(); var weights = await Task.Run(() => YoloCheckpoint.Load(path, network, cancellation), cancellation);
        await GameTask.MainThread(); var detector = new YoloGpuDetector(network, weights);
        try
        {
            var boxes = new YoloBox[YoloNetwork.Predictions]; var count = await detector.DetectAsync(pixels, width, height, boxes, cancellation);
            if (count < 1) throw new Exception("SAM frame fixture contains no detected person.");
            return boxes[0];
        }
        finally { await GameTask.MainThread(); detector.Dispose(); }
    }

    static async Task<double[]> Encode(string path, DinoShape shape, float[] rgb, float[] rays, float[] context, float[] positions, CancellationToken cancellation)
    {
        var backbone = await Task.Run(() => VisionCheckpoint.LoadSamBackbone(path, cancellation), cancellation);
        var body = await Task.Run(() => VisionCheckpoint.LoadSamImageConditioning(path, false, cancellation), cancellation);
        var hand = await Task.Run(() => VisionCheckpoint.LoadSamImageConditioning(path, true, cancellation), cancellation);
        await GameTask.MainThread(); var encoder = new SamGpuImageEncoder(backbone, body, hand, shape);
        try
        {
            var timer = Stopwatch.StartNew(); await encoder.RunAsync(rgb, rays, context, false, cancellation);
            var first = timer.Elapsed.TotalMilliseconds; var previous = context.ToArray();
            await GameTask.MainThread(); timer.Restart(); await encoder.RunAsync(rgb, rays, context, false, cancellation);
            var repeated = timer.Elapsed.TotalMilliseconds;
            if (!previous.SequenceEqual(context)) throw new Exception("SAM image encoding did not repeat exactly.");
            encoder.CopyPositions(positions); return new[] { first, repeated };
        }
        finally { await GameTask.MainThread(); encoder.Dispose(); }
    }
}