Editor/Gpu/SamGpuFeatures.cs
using System;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Sandbox;
namespace HumanoidMocapX.Inference;
/// <summary>Extracts FlowHMR's body, left-hand and right-hand features from one
/// tracked person. Owns the shared image encoder and both pose decoders.</summary>
internal sealed class SamGpuFeatures : IDisposable
{
internal const int FeatureWidth = 1024, Elements = FeatureWidth * 3;
readonly SamGpuImageEncoder encoder;
readonly SamGpuPoseDecoder body, hand;
readonly SamHandBoxes handBoxes;
readonly DinoShape shape;
readonly SamRays rayEncoder;
readonly float[] pixels, rays, context, positions, affine = new float[6], camera = new float[9], boxes = new float[8];
readonly float[] points = new float[3 * SamKeypoints.Count * 2];
readonly Action prepare, detectHands;
ReadOnlyMemory<byte> source;
SamCrop crop;
SamProjection projection;
CancellationToken cancellation;
int width, height, pass;
bool running, disposed;
internal ReadOnlySpan<float> HandBoxes => boxes;
/// <summary>Original-image XY landmarks for body, mirrored-left and right
/// passes. Hand passes use landmarks 21–41; other hand landmarks are masked.</summary>
internal ReadOnlySpan<float> ImageKeypoints => points;
internal SamGpuFeatures(SamGpuImageEncoder encoder, SamGpuPoseDecoder body, SamGpuPoseDecoder hand, SamHandBoxes handBoxes, DinoShape shape)
{
ThreadSafe.AssertIsMainThread(); ArgumentNullException.ThrowIfNull(encoder); ArgumentNullException.ThrowIfNull(body);
ArgumentNullException.ThrowIfNull(hand); ArgumentNullException.ThrowIfNull(handBoxes);
if (!shape.IsSam || shape.Width != shape.Height || shape.Channels != 1280) throw new ArgumentException("FlowHMR requires square SAM crops with 1280 image channels.");
this.encoder = encoder; this.body = body; this.hand = hand; this.handBoxes = handBoxes; this.shape = shape;
rayEncoder = new(shape); pixels = new float[shape.ImageElements]; rays = new float[rayEncoder.Elements];
context = new float[shape.Patches * shape.Channels]; positions = new float[context.Length];
prepare = Prepare; detectHands = DetectHands;
}
/// <summary>Writes 3072 channels in body/left/right order, matching FlowHMR's
/// first-pass decoder hooks, and optionally the frame's <see cref="SamObservations"/>.
/// Requires original RGBA8 pixels, XYXY person box and original-pixel intrinsics.
/// Discard output on failure; keep inputs unchanged until completion. No download or
/// external process is used.</summary>
internal async Task RunAsync(ReadOnlyMemory<byte> rgba, int width, int height, ReadOnlyMemory<float> personBox,
ReadOnlyMemory<float> intrinsics, Memory<float> output, CancellationToken token = default, Memory<float> observations = default)
{
ThreadSafe.AssertIsMainThread(); ObjectDisposedException.ThrowIf(disposed, this);
if (running) throw new InvalidOperationException("Await current SAM features before starting another frame.");
token.ThrowIfCancellationRequested();
if (width < 1 || height < 1 || width >= short.MaxValue || height >= short.MaxValue || rgba.Length != (long)width * height * 4
|| personBox.Length != 4 || intrinsics.Length != 9 || output.Length != Elements || personBox.Span.Overlaps(output.Span)
|| intrinsics.Span.Overlaps(output.Span) || rgba.Span.Overlaps(MemoryMarshal.AsBytes(output.Span))
|| observations.Length is not (0 or SamObservations.Elements))
throw new ArgumentException("SAM feature frame, box, intrinsics or output dimensions are invalid or overlap.");
var box = personBox.Span;
crop = new(box[0], box[1], box[2], box[3], shape); intrinsics.Span.CopyTo(camera);
source = rgba; this.width = width; this.height = height; cancellation = token; running = true;
try
{
for (pass = 0; pass < 3; pass++)
{
await Task.Run(prepare, token); await GameTask.MainThread();
await encoder.RunAsync(pixels, rays, context, pass != 0, token); encoder.CopyPositions(positions, pass != 0);
var decoder = pass == 0 ? body : hand;
await decoder.RunAsync(crop, projection, context, positions, token);
decoder.Tokens.Slice(0, FeatureWidth).CopyTo(output.Span.Slice(pass * FeatureWidth, FeatureWidth));
var pointOutput = points.AsSpan(pass * SamKeypoints.Count * 2, SamKeypoints.Count * 2);
decoder.ImageKeypoints.CopyTo(pointOutput);
if (pass == 1) for (var i = 0; i < pointOutput.Length; i += 2) pointOutput[i] = width - pointOutput[i] - 1;
if (!observations.IsEmpty) Observe(decoder, observations.Span);
if (pass == 0) { await Task.Run(detectHands, token); await GameTask.MainThread(); }
}
if (!observations.IsEmpty) points.CopyTo(observations.Span.Slice(SamObservations.ImageKeypoints));
}
finally { await GameTask.MainThread(); running = false; source = default; cancellation = default; }
}
void Prepare()
{
if (pass != 0) crop = SamHandBoxes.Crop(boxes.AsSpan((pass - 1) * 4, 4), shape, width, pass == 1);
crop.CopyAffine(affine); projection = new(camera, affine, crop.CenterX, crop.CenterY, crop.Size, shape, pass != 0);
crop.Prepare(source.Span, width, height, pixels, cancellation, pass == 1);
// Upstream keeps the original camera intrinsics even for the mirrored left image.
rayEncoder.Prepare(camera, affine, rays, cancellation);
}
void Observe(SamGpuPoseDecoder decoder, Span<float> observations)
{
if (pass == 0)
{
decoder.Keypoints.CopyTo(observations.Slice(SamObservations.BodyKeypoints));
decoder.CameraTranslation.CopyTo(observations.Slice(SamObservations.Translation));
return;
}
// Hand passes see one right hand; the left pass's image was mirrored, so its X axis flips back.
var hand = decoder.Keypoints.Slice(SamObservations.RightHandStart * 3, SamObservations.HandPoints * 3);
var target = observations.Slice(pass == 1 ? SamObservations.LeftHand : SamObservations.RightHand, hand.Length);
hand.CopyTo(target);
if (pass == 1) for (var i = 0; i < target.Length; i += 3) target[i] = -target[i];
}
void DetectHands() => handBoxes.Run(body.Tokens.Slice(SamInitialTokens.HandBoxStart * FeatureWidth, FeatureWidth * 2), crop, boxes, cancellation);
public void Dispose()
{
ThreadSafe.AssertIsMainThread();
if (disposed) return;
if (running) throw new InvalidOperationException("Await SAM frame inference before disposal.");
disposed = true; hand.Dispose(); body.Dispose(); encoder.Dispose();
}
}