Editor/SamFeatures.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using HumanoidMocapX.Inference;
using Sandbox;
namespace HumanoidMocapX.EditorTools;
/// <summary>Loads installed numeric SAM/MHR assets and constructs the editor's GPU
/// feature extractor. Downloads are a separate, explicit editor action.</summary>
internal static class SamFeatures
{
/// <summary>Reads checkpoints off the editor thread and builds one extractor that owns
/// all of its GPU weights. The caller owns the returned extractor.</summary>
internal static async Task<SamGpuFeatures> LoadAsync(string samPath, string mhrPath, Action<string> progress, CancellationToken cancellation = default)
=> (await LoadAsync(samPath, mhrPath, 1, null, progress, cancellation))[0];
/// <summary>Reads checkpoints once, off the editor thread, and builds independent
/// extractors that share DINOv3, decoder and head weights; each keeps its own buffers
/// and MHR body. GPU weights come from <paramref name="shared"/> when given, which the
/// caller disposes after every extractor. Reports loading stages and releases GPU
/// allocations on failure or cancellation. The caller owns the returned extractors.</summary>
internal static async Task<SamGpuFeatures[]> LoadAsync(string samPath, string mhrPath, int count, FlowGpuWeights shared, Action<string> progress,
CancellationToken cancellation = default)
{
ArgumentOutOfRangeException.ThrowIfLessThan(count, 1);
var shape = DinoShape.Sam(); var decoderShape = new SamDecoderShape(SamInitialTokens.Count, shape.Patches);
var extractors = new SamGpuFeatures[count];
SamGpuImageEncoder image = null; SamGpuPoseDecoder body = null, hand = null;
try
{
progress?.Invoke("Loading SAM image encoder");
var backbone = await Task.Run(() => VisionCheckpoint.LoadSamBackbone(samPath, cancellation), cancellation);
var bodyImage = await Task.Run(() => VisionCheckpoint.LoadSamImageConditioning(samPath, false, cancellation), cancellation);
var handImage = await Task.Run(() => VisionCheckpoint.LoadSamImageConditioning(samPath, true, cancellation), cancellation);
progress?.Invoke("Loading SAM decoders");
var bodyBranch = await Task.Run(() => new Branch(samPath, false, cancellation), cancellation);
var handBranch = await Task.Run(() => new Branch(samPath, true, cancellation), cancellation);
var boxWeights = await Task.Run(() => VisionCheckpoint.LoadSamHandBoxes(samPath, cancellation), cancellation);
// Lanes share the first body's ~664 MB corrective projection; everything mutable stays per lane.
MhrCorrectives correctives = null;
for (var lane = 0; lane < count; lane++)
{
progress?.Invoke("Loading SAM body model");
var model = await Task.Run(() => MhrCheckpoint.LoadBody(mhrPath, correctives, cancellation), cancellation);
correctives ??= model.Correctives;
var mapped = new SortedSet<int>();
await GameTask.MainThread(); cancellation.ThrowIfCancellationRequested();
image = new(backbone, bodyImage, handImage, shape, shared);
body = await Build(bodyBranch, model, mapped); hand = await Build(handBranch, model, mapped);
// Both heads read the mesh only through their keypoint mappings, so correcting the
// mapped vertices gives identical keypoints at a fraction of the projection cost.
model.Restrict(mapped.ToArray());
var boxes = new SamHandBoxes(boxWeights);
await GameTask.MainThread(); cancellation.ThrowIfCancellationRequested();
extractors[lane] = new(image, body, hand, boxes, shape); image = null; body = hand = null;
}
return extractors;
}
catch
{
await GameTask.MainThread(); hand?.Dispose(); body?.Dispose(); image?.Dispose();
foreach (var extractor in extractors) extractor?.Dispose();
throw;
}
async Task<SamGpuPoseDecoder> Build(Branch branch, MhrBody model, SortedSet<int> mapped)
{
var pose = await Task.Run(() => new SamPoseHead(new(branch.Head, branch.IsHand), SamCheckpoint.LoadParameters(samPath, branch.Head, branch.IsHand, cancellation),
new(branch.Head, branch.IsHand), model), cancellation);
mapped.UnionWith(pose.MappedVertices());
var camera = new SamPredictionHead(branch.Head, branch.IsHand, true);
var keypoints = new SamKeypointFeedback(branch.Feedback, decoderShape, shape.Width / DinoShape.PatchSize, shape.Height / DinoShape.PatchSize, branch.IsHand);
var initial = new SamInitialTokens(branch.Initial, branch.IsHand);
await GameTask.MainThread(); cancellation.ThrowIfCancellationRequested();
return new(branch.Decoder, decoderShape, initial, new(decoderShape, pose, camera, keypoints), branch.IsHand, shared);
}
}
/// <summary>One body or hand branch's checkpoint tensors, read once and shared by every lane.</summary>
sealed class Branch(string samPath, bool isHand, CancellationToken cancellation)
{
internal readonly bool IsHand = isHand;
internal readonly FlowWeights Head = VisionCheckpoint.LoadSamHead(samPath, isHand, cancellation);
internal readonly FlowWeights Feedback = VisionCheckpoint.LoadSamFeedback(samPath, isHand, cancellation);
internal readonly FlowWeights Initial = VisionCheckpoint.LoadSamInitialTokens(samPath, isHand, cancellation);
internal readonly FlowWeights Decoder = VisionCheckpoint.LoadSamDecoder(samPath, isHand, cancellation);
}
}