Editor/Gpu/FlowGpuSampler.cs
// C# adaptation of FlowHMR's HunyuanMotionMMDiT and x1 sampler. See THIRD_PARTY_NOTICES.md.
using System;
using System.Threading;
using System.Threading.Tasks;
using Sandbox;
using HumanoidMocapX.TensorInference;
namespace HumanoidMocapX.Inference;
/// <summary>Runs all twenty FlowHMR steps with GPU-resident weights and working tensors.
/// Create, sample and dispose on the main thread. Instances support one capture at a time.</summary>
internal sealed class FlowGpuSampler : IDisposable
{
readonly FlowGpuStorage storage = new();
readonly TensorBatch batch;
readonly FlowGpuWorkspace workspace;
readonly FlowGpuLinear inputEncoder, featureEncoder, cameraEncoder, finalModulation, finalProjection;
readonly FlowGpuMlp timeEncoder;
readonly FlowGpuRefiner refiner;
readonly FlowGpuDoubleBlock[] doubles;
readonly FlowGpuSingleBlock[] singles;
readonly GpuBuffer<float> body, camera, context, features, state, prediction, nextState;
readonly GpuBuffer<float> mainTimes, refinerTimes, timeInput, timeCondition, textCondition, adapter, activatedAdapter;
readonly Action recordStep;
readonly float[] fence = new float[1];
readonly int frames, width, latentElements;
CancellationToken recordingCancellation;
int step;
bool running, disposed;
internal FlowGpuSampler(FlowWeights weights, int frames, int width = FlowDenoiser.ModelWidth,
int heads = FlowDenoiser.ModelHeads)
{
ThreadSafe.AssertIsMainThread();
ArgumentNullException.ThrowIfNull(weights);
if (frames < 1 || frames > FlowRotary.MaximumFrames || width < FlowGpuWorkspace.HeadWidth
|| width > FlowDenoiser.ModelWidth || heads < 1 || (long)heads * FlowGpuWorkspace.HeadWidth != width)
throw new ArgumentException("GPU FlowHMR needs 1–5000 frames and 64-channel attention heads.");
if (!Graphics.IsAvailable) throw new InvalidOperationException("GPU FlowHMR requires an initialized graphics device.");
this.frames = frames; this.width = width; latentElements = checked(frames * FlowSampler.Channels);
try
{
workspace = new(storage, frames, width, heads);
inputEncoder = new(storage, weights, "input_encoder", FlowSampler.Channels, width);
featureEncoder = new(storage, weights, "ctxt_encoder.feature", FlowDenoiser.FeatureChannels, width);
cameraEncoder = new(storage, weights, "ctxt_encoder.camera_R", FlowDenoiser.CameraChannels, width);
timeEncoder = new(storage, weights, "timestep_encoder.blocks", width);
refiner = new(storage, weights, workspace);
finalModulation = new(storage, weights, "final_layer.adaLN_modulation.linear", width, 2 * width);
finalProjection = new(storage, weights, "final_layer.linear", width, FlowSampler.Channels);
doubles = new FlowGpuDoubleBlock[FlowDenoiser.DoubleBlocks]; singles = new FlowGpuSingleBlock[FlowDenoiser.SingleBlocks];
for (var i = 0; i < doubles.Length; i++) doubles[i] = new(storage, weights, i, width);
for (var i = 0; i < singles.Length; i++) singles[i] = new(storage, weights, i, width);
body = storage.Create<float>(frames * FlowDenoiser.FeatureChannels);
camera = storage.Create<float>(frames * FlowDenoiser.CameraChannels);
context = storage.Create<float>(workspace.StreamElements); features = storage.Create<float>(2 * workspace.StreamElements);
state = storage.Create<float>(latentElements); prediction = storage.Create<float>(latentElements);
nextState = storage.Create<float>(latentElements);
mainTimes = Times(1000); refinerTimes = Times(1);
timeInput = storage.Create<float>(width); timeCondition = storage.Create<float>(width);
adapter = storage.Create<float>(width); activatedAdapter = storage.Create<float>(width);
var text = new float[width];
new FlowConditionMlp(weights, "vtxt_encoder.linears", FlowDenoiser.TextChannels, width)
.Run(new float[FlowDenoiser.TextChannels], text, default);
textCondition = storage.Upload<float>(text);
recordStep = RecordStep;
batch = new TensorBatch();
}
catch { storage.Dispose(); throw; }
}
/// <summary>Consumes [frames,3072] body features, [frames,9] relative camera rotations
/// and [frames,491] initial noise. Writes sampled latents to output; discard output on
/// failure or cancellation. Inputs are uploaded before the first asynchronous wait.</summary>
internal async Task SampleAsync(ReadOnlyMemory<float> bodyFeatures, ReadOnlyMemory<float> cameraRotations,
ReadOnlyMemory<float> initialNoise, Memory<float> output, CancellationToken cancellation = default)
{
ThreadSafe.AssertIsMainThread();
ObjectDisposedException.ThrowIf(disposed, this);
if (running) throw new InvalidOperationException("Await the current GPU capture before sampling again.");
cancellation.ThrowIfCancellationRequested();
Validate(bodyFeatures.Span, frames * FlowDenoiser.FeatureChannels, nameof(bodyFeatures));
Validate(cameraRotations.Span, frames * FlowDenoiser.CameraChannels, nameof(cameraRotations));
Validate(initialNoise.Span, latentElements, nameof(initialNoise));
if (output.Length != latentElements) throw new ArgumentException("GPU output must have [frames,491] elements.", nameof(output));
running = true; recordingCancellation = cancellation;
Exception failure = null;
try
{
body.SetData<float>(bodyFeatures.Span); camera.SetData<float>(cameraRotations.Span); state.SetData<float>(initialNoise.Span);
for (step = 0; step < FlowSampler.Steps; step++)
{
cancellation.ThrowIfCancellationRequested();
var destination = step == FlowSampler.Steps - 1 ? output : fence.AsMemory();
// Fence each step to bound submissions and make cancellation observable without reading every tensor.
await batch.RunAsync(recordStep, state, destination, cancellation);
await GameTask.MainThread();
Validate(destination.Span, destination.Length, "sampled motion");
}
}
catch (Exception error) { failure = error; }
await GameTask.MainThread();
running = false; recordingCancellation = default;
if (failure is OperationCanceledException && cancellation.IsCancellationRequested) cancellation.ThrowIfCancellationRequested();
if (failure is not null) throw new InvalidOperationException("GPU FlowHMR sampling failed; discard the partial output.", failure);
}
GpuBuffer<float> Times(float factor)
{
var values = new float[FlowSampler.Steps * width];
for (var index = 0; index < FlowSampler.Steps; index++)
FlowConditioning.Time(index / (float)FlowSampler.Steps, factor, values.AsSpan(index * width, width));
return storage.Upload<float>(values);
}
void RecordStep()
{
recordingCancellation.ThrowIfCancellationRequested();
if (step == 0) SetVideo();
var kernels = workspace.Kernels;
inputEncoder.Run(kernels, state, features, frames);
mainTimes.CopyTo(timeInput, step * width, 0, width);
timeEncoder.Run(workspace, timeInput, timeCondition);
kernels.Add(timeCondition, textCondition, adapter, width);
kernels.Activate(adapter, activatedAdapter, width, TensorActivation.Silu);
refinerTimes.CopyTo(timeInput, step * width, 0, width);
refiner.Run(workspace, timeInput, features, recordingCancellation);
foreach (var block in doubles)
{ recordingCancellation.ThrowIfCancellationRequested(); block.Run(workspace, features, activatedAdapter); }
foreach (var block in singles)
{ recordingCancellation.ThrowIfCancellationRequested(); block.Run(workspace, features, activatedAdapter); }
finalModulation.Run(kernels, activatedAdapter, workspace.MotionGates);
kernels.Normalize(features, null, null, workspace.Normalized, frames, width, 1e-5f);
kernels.Modulate(workspace.Normalized, workspace.MotionGates, workspace.Modified, frames, width, 0, width);
finalProjection.Run(kernels, workspace.Modified, prediction, frames);
var time = step / (float)FlowSampler.Steps;
var delta = (step + 1) / (float)FlowSampler.Steps - time;
kernels.Blend(state, prediction, nextState, latentElements, delta / Math.Max(1 - time, 1f / FlowSampler.Steps));
nextState.CopyTo(state, latentElements);
}
void SetVideo()
{
featureEncoder.Run(workspace.Kernels, body, context, frames);
cameraEncoder.Run(workspace.Kernels, camera, workspace.Normalized, frames);
workspace.Kernels.Add(context, workspace.Normalized, workspace.Modified, workspace.StreamElements);
workspace.Modified.CopyTo(context, workspace.StreamElements);
refiner.SetVideo(workspace, context);
}
static void Validate(ReadOnlySpan<float> values, int length, string name)
{
if (values.Length != length) throw new ArgumentException("GPU FlowHMR tensor shape mismatch: " + name);
foreach (var value in values)
if (!float.IsFinite(value)) throw new ArithmeticException("Non-finite GPU FlowHMR tensor: " + name);
}
/// <summary>Releases weights, scratch storage and the offscreen view. Await any
/// in-flight capture first, including one whose cancellation has been requested.</summary>
public void Dispose()
{
ThreadSafe.AssertIsMainThread();
if (disposed) return;
if (running) throw new InvalidOperationException("Await GPU capture completion before disposing the sampler.");
disposed = true; batch.Dispose(); storage.Dispose();
}
}