CharacterKit/FaceMeasurement.cs

FaceMeasurement parses and validates a face-analysis JSON payload and stores measured ratios, pose, yaw, profile points and quality. It normalizes profile points, computes yaw from a pose matrix, and enforces many sanity checks about photo size, landmarks, mouth openness and pose.

File Access
using System;
using System.Text.Json;

namespace CharacterEditor;
/// <summary>One camera view. Screen points and pose stay separate until the joint fit.</summary>
public sealed class FaceMeasurement
{
	public Dictionary<string, float> Ratios { get; set; } = new();
	public float Quality { get; set; } = 1;
	public float Yaw { get; set; }
	public float[] Pose { get; set; }
	public Vector2[] Profile { get; set; }
	public bool FrontUsable => Yaw <= 30;
	public bool ProfileUsable => Pose is not null && Profile is not null && Yaw >= 20;
	public string Description => ProfileUsable ? (FrontUsable ? "Front + depth" : Yaw < 65 ? "Angled · depth" : "Side · depth") + $" ({Yaw:0}°)" : "Front · widths";

	public static Vector2[] NormalizeProfile(Vector2[] points, float[] pose)
	{
		var down = points[6] - points[1];
		float length = down.Length;
		if (length < .0001f)
			throw new Exception("Could not measure the lower face.");
		down /= length;
		var side = new Vector2(down.y, -down.x);
		if (Vector2.Dot(side, new Vector2(pose[8], -pose[9])) < 0)
			side = -side;
		return points.Select(p => new Vector2(Vector2.Dot(p - points[1], side) / length, Vector2.Dot(p - points[1], down) / length)).ToArray();
	}

	public static FaceMeasurement Parse(string json)
	{
		if (json is null || json.Length > 16000)
			throw new Exception("Invalid face result.");
		using var doc = JsonDocument.Parse(json);
		var root = doc.RootElement;
		if (root.GetProperty("schema").GetString() != "sbox.face-analysis" || root.GetProperty("version").GetInt32() != 1 || root.GetProperty("kind").GetString() != "analysis")
			throw new Exception("Unsupported face result.");
		var result = new FaceMeasurement();
		foreach (var key in PhotoFitAnchors.Keys.Append("mouthOpening"))
		{
			var value = root.GetProperty("ratios").GetProperty(key).GetSingle();
			if (!float.IsFinite(value) || value < 0 || value > 100)
				throw new Exception("Invalid face proportions.");
			result.Ratios[key] = value;
		}

		var landmarks = root.GetProperty("landmarks").EnumerateArray().ToDictionary(x => x[0].GetInt32(), x => new Vector2(x[1].GetSingle(), x[2].GetSingle()));
		if (landmarks.Values.Any(x => !float.IsFinite(x.x) || !float.IsFinite(x.y) || MathF.Abs(x.x) > 4 || MathF.Abs(x.y) > 4))
			throw new Exception("Invalid landmarks.");
		var size = root.GetProperty("imageSize");
		float w = size[0].GetSingle(), h = size[1].GetSingle();
		if (!float.IsFinite(w) || !float.IsFinite(h) || w <= 0 || h <= 0 || w > 4096 || h > 4096)
			throw new Exception("Invalid photo size.");
		Vector2 P(int id) => new(landmarks[id].x * w, landmarks[id].y * h);
		if (root.TryGetProperty("poseMatrix", out var matrix) && matrix.ValueKind == JsonValueKind.Array)
		{
			var m = matrix.EnumerateArray().Select(x => x.GetSingle()).ToArray();
			if (m.Length != 16 || m.Any(x => !float.IsFinite(x)))
				throw new Exception("Invalid camera pose.");
			// MediaPipe exports column-major canonical-to-camera transforms. Only rotation
			// is used: screen landmarks aren't metric coordinates, so don't inverse-map z.
			for (int c = 0; c < 3; c++)
			{
				float length = MathF.Sqrt(m[c * 4] * m[c * 4] + m[c * 4 + 1] * m[c * 4 + 1] + m[c * 4 + 2] * m[c * 4 + 2]);
				if (length < .1f || length > 10)
					throw new Exception("Unreliable camera pose.");
				for (int r = 0; r < 3; r++)
					m[c * 4 + r] /= length;
			}

			Vector3 Column(int c) => new(m[c * 4], m[c * 4 + 1], m[c * 4 + 2]);
			if (MathF.Abs(Vector3.Dot(Column(0), Column(1))) > .1f || MathF.Abs(Vector3.Dot(Column(0), Column(2))) > .1f || MathF.Abs(Vector3.Dot(Column(1), Column(2))) > .1f || Vector3.Dot(Vector3.Cross(Column(0), Column(1)), Column(2)) < .8f)
				throw new Exception("Unreliable camera pose.");
			var up = new Vector2(m[4], -m[5]);
			if (up.Length < .6f)
				throw new Exception("Look closer to level for a reliable fit.");
			var side = new Vector2(-up.y, up.x).Normal;
			result.Yaw = MathF.Atan2(MathF.Abs(Vector2.Dot(new(m[8], -m[9]), side)), m[10]) * 180 / MathF.PI;
			if (result.Yaw > 100)
				throw new Exception("The face is turned away from the camera.");
			result.Pose = m;
			var profile = PhotoProfileAnchors.Ids.Select(P).ToArray();
			float lowerHeight = (P(152) - P(168)).Length;
			if (lowerHeight < 40)
				throw new Exception("Face is too small. Use a closer photo.");
			if ((P(13) - P(14)).Length / lowerHeight > .13f)
				throw new Exception("Please use a relaxed, closed-mouth photo.");
			result.Profile = NormalizeProfile(profile, m);
			if (result.Profile.Any(x => MathF.Abs(x.x) > 2 || MathF.Abs(x.y) > 4))
				throw new Exception("Face landmarks are inconsistent. Try a clearer photo.");
			result.Quality = Math.Clamp(lowerHeight / 160, .4f, 1);
			// Widths are never inferred from a silhouette. Moderate front-view vertical
			// ratios compensate for the cheek denominator foreshortening with yaw.
			if (result.FrontUsable)
			{
				float cosine = MathF.Cos(result.Yaw * MathF.PI / 180);
				result.Ratios["faceHeight"] *= cosine;
				result.Ratios["noseLength"] *= cosine;
			}
		}
		else
		{
			// Legacy front-only bridge payloads remain usable; cannot infer profile pose.
			var axis = (P(454) - P(234)).Normal;
			float width = (P(454) - P(234)).Length;
			if (width < 40)
				throw new Exception("Face is too small. Use a closer photo.");
			float yaw = MathF.Abs(Vector2.Dot(P(1) - (P(234) + P(454)) * .5f, axis)) / width;
			if (yaw > .18f)
				throw new Exception("This angled reference has no camera pose. Reload it with the current photo analyzer.");
			if (result.Ratios["mouthOpening"] > .055f)
				throw new Exception("Please use a relaxed, closed-mouth photo.");
			result.Quality = Math.Clamp(1 - yaw * 3, .2f, 1);
		}

		if (result.FrontUsable && (result.Ratios["faceHeight"] < .7f || result.Ratios["noseWidth"] < .06f || result.Ratios["mouthWidth"] < .1f))
			throw new Exception("Front-view landmarks are obscured. Try a clearer photo.");
		return result;
	}
}