Editor/Gpu/SamGpuImageEncoder.cs
// C# adaptation of SAM 3D Body's image branch. See LICENSE-SAM and THIRD_PARTY_NOTICES.md.
// Copyright (c) Meta Platforms, Inc. and affiliates.
using System;
using System.Threading;
using System.Threading.Tasks;
using Sandbox;
using HumanoidMocapX.TensorInference;

namespace HumanoidMocapX.Inference;

/// <summary>Owns one shared DINOv3 encoder and separate body/hand image conditioning.
/// Create, run and dispose on the main thread; crop and ray preparation stay pure C#.</summary>
internal sealed class SamGpuImageEncoder : IDisposable
{
    readonly FlowGpuStorage storage;
    readonly DinoShape shape;
    readonly DinoGpuEncoder encoder;
    readonly OmegaGpuWorkspace workspace;
    readonly SamGpuImageConditioning bodyConditioning, handConditioning;
    readonly TensorBatch batch;
    readonly GpuBuffer<float> pixels, features, rays, context;
    readonly float[] normalized, bodyPositions, handPositions;
    readonly Action prepare, record;
    ReadOnlyMemory<float> inputImage, inputRays;
    Memory<float> destination;
    CancellationToken cancellation;
    bool isHand, running, disposed;

    internal SamGpuImageEncoder(FlowWeights backbone, FlowWeights body, FlowWeights hand, DinoShape shape, FlowGpuWeights shared = null)
    {
        ThreadSafe.AssertIsMainThread();
        if (!shape.IsSam) throw new ArgumentException("SAM image encoding requires a SAM DINO shape.");
        this.shape = shape; storage = new(shared);
        try
        {
            workspace = new(storage, shape.Tokens, shape.Channels, true); encoder = new(storage, backbone, shape);
            bodyConditioning = new(storage, body, shape); handConditioning = new(storage, hand, shape, true);
            bodyPositions = SamImageConditioning.Positions(body, shape, false); handPositions = SamImageConditioning.Positions(hand, shape, true);
            pixels = storage.Create<float>(shape.ImageElements); normalized = new float[shape.ImageElements];
            features = storage.Create<float>(shape.Patches * shape.Channels); context = storage.Create<float>(features.ElementCount);
            rays = storage.Create<float>(shape.Patches * SamRays.Channels);
            prepare = Prepare; record = Record; batch = new();
        }
        catch { storage.Dispose(); throw; }
    }

    /// <summary>Encodes RGB [0,1] and camera-ray features into the decoder context.
    /// Keeps all inputs unchanged until completion. Rejects invalid dimensions,
    /// aliasing, non-finite values, concurrent calls and use after disposal.</summary>
    internal async Task RunAsync(ReadOnlyMemory<float> image, ReadOnlyMemory<float> rayFeatures, Memory<float> output,
        bool hand = false, CancellationToken token = default)
    {
        ThreadSafe.AssertIsMainThread(); ObjectDisposedException.ThrowIf(disposed, this);
        if (running) throw new InvalidOperationException("Await the current SAM image encoding before starting another.");
        token.ThrowIfCancellationRequested(); running = true; cancellation = token; isHand = hand;
        inputImage = image; inputRays = rayFeatures; destination = output;
        try
        {
            await Task.Run(prepare, token); await GameTask.MainThread();
            pixels.SetData<float>(normalized); rays.SetData<float>(inputRays.Span);
            await batch.RunAsync(record, context, destination, token);
            SamDecoder.Finite(destination.Span);
        }
        finally
        {
            await GameTask.MainThread(); running = false; cancellation = default;
            inputImage = inputRays = default; destination = default;
        }
    }

    /// <summary>Copies fixed positional image context for the selected decoder.</summary>
    internal void CopyPositions(Span<float> output, bool hand = false)
    {
        ObjectDisposedException.ThrowIf(disposed, this);
        if (output.Length != bodyPositions.Length) throw new ArgumentException("SAM positional context has the wrong size.");
        (hand ? handPositions : bodyPositions).AsSpan().CopyTo(output);
    }

    void Prepare()
    {
        if (inputImage.Length != shape.ImageElements || inputRays.Length != shape.Patches * SamRays.Channels
            || destination.Length != shape.Patches * shape.Channels || inputImage.Span.Overlaps(destination.Span) || inputRays.Span.Overlaps(destination.Span))
            throw new ArgumentException("SAM image encoder inputs/output have invalid dimensions or overlap.");
        cancellation.ThrowIfCancellationRequested(); SamDecoder.Finite(inputRays.Span);
        DinoImageEncoder.NormalizeImage(inputImage.Span, normalized);
    }

    void Record()
    {
        encoder.Run(workspace, pixels, features, cancellation);
        (isHand ? handConditioning : bodyConditioning).Run(workspace.Kernels, features, rays, context, cancellation);
    }

    public void Dispose()
    {
        ThreadSafe.AssertIsMainThread();
        if (disposed) return;
        if (running) throw new InvalidOperationException("Await SAM image encoding and readback before disposal.");
        disposed = true; batch.Dispose(); storage.Dispose();
    }
}