Code/CharacterKit/FaceMeasurement.cs

A data class that parses and validates face-analysis JSON and converts facial landmarks into normalized measurements and profile points. It reads ratios, landmarks, optional pose matrix, computes yaw, normalizes profile points, and applies validation and quality heuristics.

File Access
using System;
using System.Text.Json;

namespace CharacterEditor;
/// <summary>One camera view. Screen points and pose stay separate until the joint fit.</summary>
public sealed class FaceMeasurement
{
	public Dictionary<string, float> Ratios { get; set; } = new();
	public float Quality { get; set; } = 1;
	public float Yaw { get; set; }
	public float[] Pose { get; set; }
	public Vector2[] Profile { get; set; }
	public bool FrontUsable => Yaw <= 30;
	public bool ProfileUsable => Pose is not null && Profile is not null && Yaw >= 20;
	public string Description => ProfileUsable ? (FrontUsable ? "Front + depth" : Yaw < 65 ? "Angled · depth" : "Side · depth") + $" ({Yaw:0}°)" : "Front · widths";

	public static Vector2[] NormalizeProfile(Vector2[] points, float[] pose)
	{
		var down = points[6] - points[1];
		float length = down.Length;
		if (length < .0001f)
			throw new Exception("Could not measure the lower face.");
		down /= length;
		var side = new Vector2(down.y, -down.x);
		if (Vector2.Dot(side, new Vector2(pose[8], -pose[9])) < 0)
			side = -side;
		return points.Select(p => new Vector2(Vector2.Dot(p - points[1], side) / length, Vector2.Dot(p - points[1], down) / length)).ToArray();
	}

	public static FaceMeasurement Parse(string json)
	{
		if (json is null || json.Length > 16000)
			throw new Exception("Invalid face result.");
		using var doc = JsonDocument.Parse(json);
		var root = doc.RootElement;
		if (root.GetProperty("schema").GetString() != "sbox.face-analysis" || root.GetProperty("version").GetInt32() != 1 || root.GetProperty("kind").GetString() != "analysis")
			throw new Exception("Unsupported face result.");
		var result = new FaceMeasurement();
		foreach (var key in PhotoFitAnchors.Keys.Append("mouthOpening"))
		{
			var value = root.GetProperty("ratios").GetProperty(key).GetSingle();
			if (!float.IsFinite(value) || value < 0 || value > 100)
				throw new Exception("Invalid face proportions.");
			result.Ratios[key] = value;
		}

		var landmarks = root.GetProperty("landmarks").EnumerateArray().ToDictionary(x => x[0].GetInt32(), x => new Vector2(x[1].GetSingle(), x[2].GetSingle()));
		if (landmarks.Values.Any(x => !float.IsFinite(x.x) || !float.IsFinite(x.y) || MathF.Abs(x.x) > 4 || MathF.Abs(x.y) > 4))
			throw new Exception("Invalid landmarks.");
		var size = root.GetProperty("imageSize");
		float w = size[0].GetSingle(), h = size[1].GetSingle();
		if (!float.IsFinite(w) || !float.IsFinite(h) || w <= 0 || h <= 0 || w > 4096 || h > 4096)
			throw new Exception("Invalid photo size.");
		Vector2 P(int id) => new(landmarks[id].x * w, landmarks[id].y * h);
		if (root.TryGetProperty("poseMatrix", out var matrix) && matrix.ValueKind == JsonValueKind.Array)
		{
			var m = matrix.EnumerateArray().Select(x => x.GetSingle()).ToArray();
			if (m.Length != 16 || m.Any(x => !float.IsFinite(x)))
				throw new Exception("Invalid camera pose.");
			// MediaPipe exports column-major canonical-to-camera transforms. Only rotation
			// is used: screen landmarks aren't metric coordinates, so don't inverse-map z.
			for (int c = 0; c < 3; c++)
			{
				float length = MathF.Sqrt(m[c * 4] * m[c * 4] + m[c * 4 + 1] * m[c * 4 + 1] + m[c * 4 + 2] * m[c * 4 + 2]);
				if (length < .1f || length > 10)
					throw new Exception("Unreliable camera pose.");
				for (int r = 0; r < 3; r++)
					m[c * 4 + r] /= length;
			}

			Vector3 Column(int c) => new(m[c * 4], m[c * 4 + 1], m[c * 4 + 2]);
			if (MathF.Abs(Vector3.Dot(Column(0), Column(1))) > .1f || MathF.Abs(Vector3.Dot(Column(0), Column(2))) > .1f || MathF.Abs(Vector3.Dot(Column(1), Column(2))) > .1f || Vector3.Dot(Vector3.Cross(Column(0), Column(1)), Column(2)) < .8f)
				throw new Exception("Unreliable camera pose.");
			var up = new Vector2(m[4], -m[5]);
			if (up.Length < .6f)
				throw new Exception("Look closer to level for a reliable fit.");
			var side = new Vector2(-up.y, up.x).Normal;
			result.Yaw = MathF.Atan2(MathF.Abs(Vector2.Dot(new(m[8], -m[9]), side)), m[10]) * 180 / MathF.PI;
			if (result.Yaw > 100)
				throw new Exception("The face is turned away from the camera.");
			result.Pose = m;
			var profile = PhotoProfileAnchors.Ids.Select(P).ToArray();
			float lowerHeight = (P(152) - P(168)).Length;
			if (lowerHeight < 40)
				throw new Exception("Face is too small. Use a closer photo.");
			if ((P(13) - P(14)).Length / lowerHeight > .13f)
				throw new Exception("Please use a relaxed, closed-mouth photo.");
			result.Profile = NormalizeProfile(profile, m);
			if (result.Profile.Any(x => MathF.Abs(x.x) > 2 || MathF.Abs(x.y) > 4))
				throw new Exception("Face landmarks are inconsistent. Try a clearer photo.");
			result.Quality = Math.Clamp(lowerHeight / 160, .4f, 1);
			// Widths are never inferred from a silhouette. Moderate front-view vertical
			// ratios compensate for the cheek denominator foreshortening with yaw.
			if (result.FrontUsable)
			{
				float cosine = MathF.Cos(result.Yaw * MathF.PI / 180);
				result.Ratios["faceHeight"] *= cosine;
				result.Ratios["noseLength"] *= cosine;
			}
		}
		else
		{
			// Legacy front-only bridge payloads remain usable; cannot infer profile pose.
			var axis = (P(454) - P(234)).Normal;
			float width = (P(454) - P(234)).Length;
			if (width < 40)
				throw new Exception("Face is too small. Use a closer photo.");
			float yaw = MathF.Abs(Vector2.Dot(P(1) - (P(234) + P(454)) * .5f, axis)) / width;
			if (yaw > .18f)
				throw new Exception("This angled reference has no camera pose. Reload it with the current photo analyzer.");
			if (result.Ratios["mouthOpening"] > .055f)
				throw new Exception("Please use a relaxed, closed-mouth photo.");
			result.Quality = Math.Clamp(1 - yaw * 3, .2f, 1);
		}

		if (result.FrontUsable && (result.Ratios["faceHeight"] < .7f || result.Ratios["noseWidth"] < .06f || result.Ratios["mouthWidth"] < .1f))
			throw new Exception("Front-view landmarks are obscured. Try a clearer photo.");
		return result;
	}
}