Editor/Inference/SamRays.cs
// C# adaptation of SAM 3D Body's camera encoding. See LICENSE-SAM and THIRD_PARTY_NOTICES.md.
// Copyright (c) Meta Platforms, Inc. and affiliates.
using System;
using System.Threading;
namespace HumanoidMocapX.Inference;
/// <summary>Encodes crop-camera rays at DINO patch resolution without allocating a full image of rays.</summary>
internal sealed class SamRays
{
internal const int Bands = 16, Channels = 99;
readonly float[] horizontal, vertical;
internal int Elements => horizontal.Length * vertical.Length * Channels;
internal SamRays(DinoShape shape)
{
if (!shape.IsSam) throw new ArgumentException("Camera conditioning requires a SAM image shape.", nameof(shape));
horizontal = Coordinates(shape.Width); vertical = Coordinates(shape.Height);
}
/// <summary>Consumes row-major full-image intrinsics[3,3] and full-to-crop affine[2,3].
/// Writes [patches,99]. Rotation/skew, invalid cameras and overlapping buffers fail.
/// Coordinates refer to the already flipped image when preparing the left hand.</summary>
internal void Prepare(ReadOnlySpan<float> intrinsics, ReadOnlySpan<float> affine, Span<float> output,
CancellationToken cancellation = default)
{
if (intrinsics.Length != 9 || affine.Length != 6 || output.Length != Elements
|| intrinsics.Overlaps(output) || affine.Overlaps(output)) throw new ArgumentException("SAM ray tensors have invalid dimensions or overlap.");
SamDecoder.Finite(intrinsics); SamDecoder.Finite(affine); cancellation.ThrowIfCancellationRequested();
if (intrinsics[0] <= 0 || intrinsics[4] <= 0 || affine[0] <= 0 || affine[4] <= 0
|| intrinsics[1] != 0 || intrinsics[3] != 0 || intrinsics[6] != 0 || intrinsics[7] != 0 || intrinsics[8] != 1
|| affine[1] != 0 || affine[3] != 0)
throw new ArgumentException("SAM rays need positive focal/crop scales, no rotation/skew and a canonical camera last row.");
for (var y = 0; y < vertical.Length; y++)
{
cancellation.ThrowIfCancellationRequested();
for (var x = 0; x < horizontal.Length; x++)
{
var row = output.Slice((y * horizontal.Length + x) * Channels, Channels);
row[0] = (horizontal[x] / affine[0] - affine[2] / affine[0] - intrinsics[2]) / intrinsics[0];
row[1] = (vertical[y] / affine[4] - affine[5] / affine[4] - intrinsics[5]) / intrinsics[4]; row[2] = 1;
for (var axis = 0; axis < 3; axis++)
for (var band = 0; band < Bands; band++)
{
var angle = MathF.PI * (row[axis] * (1 + 31f * band / (Bands - 1)));
row[3 + axis * Bands + band] = MathF.Sin(angle);
row[3 + 3 * Bands + axis * Bands + band] = MathF.Cos(angle);
}
}
}
SamDecoder.Finite(output);
}
static float[] Coordinates(int pixels)
{
const int scale = DinoShape.PatchSize;
var coordinates = new float[pixels / scale];
// Rays are affine in pixel position; filtering coordinates preserves edge antialiasing without a 512x512 ray buffer.
for (var patch = 0; patch < coordinates.Length; patch++)
{
var center = (patch + .5f) * scale; double sum = 0, total = 0;
var start = Math.Max(0, (int)(center - scale + .5f)); var end = Math.Min(pixels, (int)(center + scale + .5f));
for (var pixel = start; pixel < end; pixel++)
{
var weight = Math.Max(0, 1 - Math.Abs((pixel + .5 - center) / scale));
sum += weight * pixel; total += weight;
}
coordinates[patch] = (float)(sum / total);
}
return coordinates;
}
}