Editor/VisionCheckpoint.cs
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using HumanoidMocapX.Inference;
namespace HumanoidMocapX.EditorTools;
internal static class VisionCheckpoint
{
/// <summary>Loads image, aggregator and camera-head tensors from a verified
/// checkpoint, excluding the unused dense-depth and text-alignment heads.</summary>
internal static FlowWeights LoadOmega(string path, CancellationToken cancellation = default)
=> Load(path, OmegaElements, cancellation, "camera_head.", "aggregator.");
/// <summary>Loads only the Omega tensors under <paramref name="prefixes"/>, for building the
/// model part by part (OmegaGpuModel.LoadAsync). Fails like LoadOmega, and when no tensor matches.</summary>
internal static FlowWeights LoadOmegaPart(string path, CancellationToken cancellation, params string[] prefixes)
=> Load(path, OmegaElements, cancellation, prefixes);
const long OmegaElements = 1_200_000_000;
/// <summary>Loads only camera-head tensors from a verified Omega checkpoint,
/// without executing pickle code. Run off the editor thread. Shapes are checked
/// when binding the head; malformed metadata and oversized tensors fail here.</summary>
internal static FlowWeights LoadOmegaCameraHead(string path, CancellationToken cancellation = default)
=> Load(path, 220_000_000, cancellation, "camera_head.");
/// <summary>Loads SAM's DINOv3 backbone from its verified checkpoint, using either
/// a root state dictionary or state_dict. Excludes decoder and MHR tensors.</summary>
internal static FlowWeights LoadSamBackbone(string path, CancellationToken cancellation = default)
=> Load(path, 900_000_000, cancellation, "backbone.encoder.");
/// <summary>Loads one SAM pose transformer's weights, excluding image and MHR
/// tensors. Binding the decoder validates every required layer and its shape.</summary>
internal static FlowWeights LoadSamDecoder(string path, bool hand, CancellationToken cancellation = default)
=> Load(path, 50_000_000, cancellation, hand ? "decoder_hand." : "decoder.");
/// <summary>Loads no-mask image conditioning and body/hand camera-position tensors.</summary>
internal static FlowWeights LoadSamImageConditioning(string path, bool hand, CancellationToken cancellation = default)
=> Load(path, 2_000_000, cancellation, hand ? "ray_cond_emb_hand." : "ray_cond_emb.",
hand ? "hand_pe_layer." : "prompt_encoder.pe_layer.", "prompt_encoder.no_mask_embed.");
/// <summary>Loads one body/hand pose and camera head, including continuous-pose
/// mappings and keypoint regression. Integer index buffers are read separately.</summary>
internal static FlowWeights LoadSamHead(string path, bool hand, CancellationToken cancellation = default)
{
var pose = hand ? "head_pose_hand." : "head_pose.";
return Load(path, 12_000_000, cancellation, pose + "proj.", pose + "scale_", pose + "hand_pose_",
pose + "keypoint_mapping", pose + "right_wrist_coords", pose + "root_coords", pose + "local_to_world_wrist",
hand ? "head_camera_hand.proj." : "head_camera.proj.",
hand ? "init_pose_hand." : "init_pose.", hand ? "init_camera_hand." : "init_camera.");
}
/// <summary>Loads the 2D/3D keypoint position MLPs and sampled-image projection.</summary>
internal static FlowWeights LoadSamFeedback(string path, bool hand, CancellationToken cancellation = default)
{
var suffix = hand ? "_hand." : ".";
return Load(path, 5_000_000, cancellation, "keypoint_posemb_linear" + suffix,
"keypoint3d_posemb_linear" + suffix, "keypoint_feat_linear" + suffix);
}
/// <summary>Loads initial pose/camera estimates, CLIFF projections, the invalid
/// point prompt and body/hand keypoint tokens for the first decoder pass.</summary>
internal static FlowWeights LoadSamInitialTokens(string path, bool hand, CancellationToken cancellation = default)
{
var suffix = hand ? "_hand." : ".";
return Load(path, 4_000_000, cancellation, "init_pose" + suffix, "init_camera" + suffix,
"init_to_token_mhr" + suffix, "prev_to_token_mhr" + suffix, "prompt_to_token.",
"prompt_encoder.invalid_point_embed.", "hand_box_embedding.",
"keypoint_embedding" + suffix, "keypoint3d_embedding" + suffix);
}
/// <summary>Loads the three-layer hand-box regressor. The classification logits
/// are unused by upstream's full inference crop selection.</summary>
internal static FlowWeights LoadSamHandBoxes(string path, CancellationToken cancellation = default)
=> Load(path, 3_000_000, cancellation, "bbox_embed.");
static FlowWeights Load(string path, long maximumElements, CancellationToken cancellation, params string[] prefixes)
{
cancellation.ThrowIfCancellationRequested();
using var checkpoint = new TorchCheckpoint(path);
var tensors = new Dictionary<string, Tensor>(StringComparer.Ordinal);
long elements = 0;
foreach (var pair in checkpoint.Tensors)
{
if (!Include(pair.Key, prefixes)) continue;
cancellation.ThrowIfCancellationRequested();
if (pair.Value.Shape.Length is < 1 or > 4) throw new InvalidDataException("Unexpected vision tensor rank: " + pair.Key);
long count = 1;
foreach (var dimension in pair.Value.Shape)
{
if (dimension < 1) throw new InvalidDataException("Empty vision tensor: " + pair.Key);
count = checked(count * dimension);
}
elements = checked(elements + count);
if (elements > maximumElements) throw new InvalidDataException("Vision weights exceed their supported size.");
}
if (elements == 0) throw new InvalidDataException("Checkpoint has no tensors for the selected vision model.");
foreach (var pair in checkpoint.Tensors)
{
if (!Include(pair.Key, prefixes)) continue;
cancellation.ThrowIfCancellationRequested();
var shape = pair.Value.Shape.AsSpan().ToArray(); Array.Reverse(shape);
tensors.Add(pair.Key, new Tensor(pair.Key, checkpoint.ReadFloat(pair.Key, cancellation), shape));
}
cancellation.ThrowIfCancellationRequested();
return new(tensors);
}
static bool Include(string name, string[] prefixes)
{
foreach (var prefix in prefixes) if (name.StartsWith(prefix, StringComparison.Ordinal)) return true;
return false;
}
}