Effigy.Tests/DmxGrammarTests.cs
using System;
using System.Collections.Generic;
using System.Linq;
using Effigy;
namespace Effigy.Tests;
/// <summary>
/// The DMX writer's output parsed as KeyValues2, rather than searched for substrings.
///
/// WHY THIS EXISTS. Every DMX check before this one asked whether a name appeared in the text —
/// "does it contain DmeModel", "does it carry jointWeights". All of them passed on a file the
/// engine's own reader rejected at line 56 with "Expecting ',', didn't find it!". Three things
/// were wrong and no substring test could see any of them:
///
/// 1. an element written INSIDE an element_array got no trailing comma, so the next member's
/// type name ran onto the previous member;
/// 2. a reference to an element defined elsewhere was written as a bare quoted id, where
/// KeyValues2 wants the two tokens "element" "<id>";
/// 3. the vertex format fields were named positions / normals / textureCoordinates /
/// jointWeights / jointIndices, where the compiler keys on position$0 / normal$0 /
/// texcoord$0 / blendweights$0 / blendindices$0.
///
/// The first two the compiler reports as "Couldn't load DMX file", the third as "Missing position
/// values" — none of which names the mistake. Note that a substring test is not merely weak here,
/// it is actively misleading: the old checks asserted the presence of the WRONG names and passed
/// for it. A render cannot see this class of bug either, because there is nothing to render. Only
/// a parse can. So this file ships a minimal KV2 reader the way SmdWriter ships a minimal SMD
/// reader, and asserts on the tree it produces.
///
/// The reader is deliberately strict — it fails where the engine fails rather than being lenient
/// and passing files the engine will not load. Its errors carry a line number for the same reason
/// dmxconvert's do: that is what made the original bug findable in one step.
/// </summary>
public static class DmxGrammarTests
{
public static void Run()
{
Report.Section( "DMX grammar: the output parses as KeyValues2" );
TestParses();
Report.Section( "DMX grammar: the parsed tree is the model that went in" );
TestContent();
Report.Section( "DMX grammar: vertex colour rides along when painted" );
TestVertexColor();
}
// --- the checks ---------------------------------------------------------------------------
static void TestParses()
{
// A rigged cylinder is the case that exercises every construct at once: nested bone dags
// inside an element_array (the comma bug), jointList full of references (the reference
// bug), face sets, and numeric arrays.
var (mesh, skeleton) = Rigged();
var text = DmxWriter.Write( mesh, skeleton, modelName: "grammar" );
var root = Parse( text, out var error );
Report.Check( "a rigged export parses", root is not null, error );
if ( root is null )
return;
// The two specific mistakes, named, so a failure says which one came back rather than
// just "does not parse".
Report.Check( "every element_array member is comma-separated",
!text.Contains( "}\n\t\t\t\"Dme" ) && !text.Contains( "}\n\t\t\"Dme" ),
"an element sits directly against the next member's type name" );
Report.Check( "references are the two-token form, not a bare id",
!System.Text.RegularExpressions.Regex.IsMatch( text, "\n\\s*\"[0-9a-f]{8}-" ),
"a bare quoted id would be read as an element type name" );
// A static export takes a different path through the writer - Skeleton.SingleRoot rather
// than a supplied one - and has to parse too.
var staticText = DmxWriter.Write( Primitives.Box( 2, 2, 2 ), modelName: "grammar_static" );
Report.Check( "a static export parses", Parse( staticText, out var staticError ) is not null, staticError );
// One bone, one face, one vertex: the degenerate end, where an off-by-one in the comma
// trimming would show up as a stray or missing separator.
var plane = Primitives.Plane( 1, 1 );
Report.Check( "a single-face export parses",
Parse( DmxWriter.Write( plane, modelName: "grammar_plane" ), out var planeError ) is not null, planeError );
// Trailing commas are trimmed at the end of every array; a file that keeps one loads in
// some readers and not others, which is the failure mode that is worst to diagnose.
Report.Check( "no array ends on a trailing comma",
!System.Text.RegularExpressions.Regex.IsMatch( text, ",\\s*\\]" ) );
}
static void TestContent()
{
var (mesh, skeleton) = Rigged();
var root = Parse( DmxWriter.Write( mesh, skeleton, modelName: "grammar" ), out var error );
if ( root is null )
{
Report.Check( "the tree is readable", false, error );
return;
}
var model = root.Element( "skeleton" );
Report.Check( "the root carries a DmeModel", model is not null && model.Type == "DmeModel" );
if ( model is null )
return;
// jointList is what the per-vertex joint indices index into, so its length is the one
// thing that makes the weights mean anything.
var joints = model.Array( "jointList" );
Report.Check( "jointList has one entry per bone",
joints.Count == skeleton.Count, $"{joints.Count} vs {skeleton.Count}" );
Report.Check( "every jointList entry is a reference",
joints.All( j => j.IsReference ), "an entry parsed as something other than a reference" );
var vertexData = FindFirst( root, "DmeVertexData" );
Report.Check( "there is a vertex data block", vertexData is not null );
if ( vertexData is null )
return;
// THE CHECK FOR THE THIRD BUG. vertexFormat declares the fields; each one then has to exist
// as an array under exactly that name. They were written as two independent sets of string
// literals and disagreed with what the compiler keys on — "positions" rather than
// "position$0" — which it reports as "Missing position values", naming a field it cannot
// find rather than the one it was given. Comparing the declaration against the arrays makes
// any future rename self-checking, whatever the names become.
var declared = vertexData.Array( "vertexFormat" ).Select( f => f.Value ).ToList();
Report.Check( "vertexFormat declares the five fields a skinned mesh needs",
declared.Count == 5, string.Join( ", ", declared ) );
Report.Check( "every declared field exists as an array under that exact name",
declared.All( f => vertexData.Array( f ).Count > 0 ),
string.Join( ", ", declared.Where( f => vertexData.Array( f ).Count == 0 ) ) );
// And the names are the $-suffixed ones, stated outright so a revert reads as a failure
// rather than as a passing test about different names.
Report.Check( "the fields are the <semantic>$<set> spelling the compiler keys on",
declared.Contains( "position$0" ) && declared.Contains( "normal$0" )
&& declared.Contains( "texcoord$0" ) && declared.Contains( "blendweights$0" )
&& declared.Contains( "blendindices$0" ),
string.Join( ", ", declared ) );
var positions = vertexData.Array( "position$0" );
var weights = vertexData.Array( "blendweights$0" );
var jointIndices = vertexData.Array( "blendindices$0" );
Report.Check( "one position per vertex",
positions.Count == mesh.VertexCount, $"{positions.Count} vs {mesh.VertexCount}" );
// The compiler states this rule outright: "Incorrect number of joint weights or indices
// specified, must match number of positions values". It is a fixed stride, so it is a
// multiplication, not an equality.
Report.Check( "weights are MaxInfluences per position",
weights.Count == mesh.VertexCount * DmxWriter.MaxInfluences,
$"{weights.Count} vs {mesh.VertexCount * DmxWriter.MaxInfluences}" );
Report.Check( "joint indices match the weights",
jointIndices.Count == weights.Count, $"{jointIndices.Count} vs {weights.Count}" );
Report.Check( "no joint index points outside the skeleton",
jointIndices.All( j => int.TryParse( j.Value, out var b ) && b >= -1 && b < skeleton.Count ) );
// Pruning to four influences has to renormalise, or a vertex that had five influences
// comes out lighter than the ones that had four and the mesh sags toward the origin.
var sums = new List<float>();
for ( var v = 0; v < mesh.VertexCount; v++ )
{
var sum = 0f;
for ( var i = 0; i < DmxWriter.MaxInfluences; i++ )
sum += float.Parse( weights[v * DmxWriter.MaxInfluences + i].Value,
System.Globalization.CultureInfo.InvariantCulture );
sums.Add( sum );
}
Report.Check( "every vertex's weights still sum to 1 after pruning",
sums.All( s => MathF.Abs( s - 1f ) < 1e-3f ),
$"worst {sums.Select( s => MathF.Abs( s - 1f ) ).Max()}" );
// The three index arrays must be the same length - the compiler says so: "Cannot add
// vertex data block with different number of normal indices (%d) and vertex indices (%d)".
var positionIndices = vertexData.Array( "position$0Indices" );
var normalIndices = vertexData.Array( "normal$0Indices" );
var uvIndices = vertexData.Array( "texcoord$0Indices" );
var corners = mesh.Faces.Sum( f => f.Count );
Report.Check( "one index per face corner",
positionIndices.Count == corners, $"{positionIndices.Count} vs {corners}" );
Report.Check( "all three index arrays are the same length",
normalIndices.Count == positionIndices.Count && uvIndices.Count == positionIndices.Count,
$"{positionIndices.Count}/{normalIndices.Count}/{uvIndices.Count}" );
// N-gons are the reason DMX is written at all. A cylinder's caps are n-gons; if the writer
// ever triangulates, the corner count above drops and this is the check that says why.
Report.Check( "the cap n-gons were not triangulated",
mesh.Faces.Any( f => f.Count > 4 ), "no face has more than four corners" );
var faceSet = FindFirst( root, "DmeFaceSet" );
Report.Check( "there is a face set", faceSet is not null );
// faces is a run of corner indices per face, each run terminated by -1. Every face in the
// mesh has to be accounted for across all the sets.
var terminators = 0;
foreach ( var set in FindAll( root, "DmeFaceSet" ) )
terminators += set.Array( "faces" ).Count( f => f.Value == "-1" );
Report.Check( "the face sets cover every face",
terminators == mesh.FaceCount, $"{terminators} vs {mesh.FaceCount}" );
}
static void TestVertexColor()
{
var (mesh, skeleton) = Rigged();
// Paint vertex 0 red, leave the rest unpainted — a painted export and an unpainted one must
// be distinguishable, and the unpainted vertices must come back WHITE, which is what leaves
// the material untouched under a multiply.
mesh.VertexColors = new Vec4[mesh.VertexCount];
mesh.VertexColors[0] = new Vec4( 1f, 0f, 0f, 1f );
var root = Parse( DmxWriter.Write( mesh, skeleton, modelName: "grammar_color" ), out var error );
Report.Check( "a painted export parses", root is not null, error );
if ( root is null )
return;
var vertexData = FindFirst( root, "DmeVertexData" );
if ( vertexData is null )
{
Report.Check( "there is a vertex data block", false );
return;
}
var declared = vertexData.Array( "vertexFormat" ).Select( f => f.Value ).ToList();
Report.Check( "vertexFormat declares color$0 when the mesh is painted",
declared.Contains( "color$0" ), string.Join( ", ", declared ) );
var colors = vertexData.Array( "color$0" );
Report.Check( "one colour per position", colors.Count == mesh.VertexCount,
$"{colors.Count} vs {mesh.VertexCount}" );
Report.Check( "colour indices mirror the position indices",
vertexData.Array( "color$0Indices" ).Count == vertexData.Array( "position$0Indices" ).Count );
// BYTES, NOT NORMALIZED FLOATS. A DMX "color" is four channels 0-255. Written as floats the
// file does not parse at all: dmxconvert says "Error reading in array attribute color$0
// element 0" and the compiler says only "Couldn't load DMX file", then "Node 'Body_LOD0'
// resolve failure". This test used to assert "1 0 0 1" and pass — which is the failure this
// file's own header describes, a check agreeing with the writer about the wrong thing.
Report.Check( "a painted vertex writes its paint colour", colors[0].Value == "255 0 0 255",
colors[0].Value );
Report.Check( "an unpainted vertex is white", colors[1].Value == "255 255 255 255",
colors[1].Value );
// The general guard, rather than two spot checks: an imported mesh carries arbitrary colour
// off the OBJ, and it was a value like "0.466482 1 0.998274 1" that actually broke the
// compile. Any fractional component anywhere in the array is the same bug.
var fractional = colors.FirstOrDefault( v => v.Value.Contains( '.' ) );
Report.Check( "no colour is written as a fraction", fractional is null,
fractional?.Value );
// And they are in range, since a byte channel that overflowed would parse and render wrong
// rather than failing loudly.
var outOfRange = colors
.SelectMany( v => v.Value.Split( ' ' ) )
.FirstOrDefault( n => !int.TryParse( n, out var b ) || b < 0 || b > 255 );
Report.Check( "every channel is a byte 0-255", outOfRange is null, outOfRange );
}
static (PolyMesh, Skeleton) Rigged()
{
var skeleton = new Skeleton();
var root = skeleton.AddBone( "root", -1, Xform.Identity, 2f );
skeleton.AddBone( "upper", root, Xform.Translate( new Vec3( 0, 2, 0 ) ), 2f );
var mesh = Primitives.Cylinder( 0.5f, 4f, 12 );
mesh.Skin = SkinBinder.BindSmooth( mesh, skeleton );
return (mesh, skeleton);
}
// --- a minimal KeyValues2 reader ------------------------------------------------------------
/// <summary>One parsed element: its type, and its attributes in order.</summary>
public sealed class Node
{
public string Type;
public string Name;
public readonly List<(string Key, Node Child)> Children = new();
public readonly Dictionary<string, string> Values = new();
public readonly Dictionary<string, List<Item>> Arrays = new();
public Node Element( string key ) =>
Children.FirstOrDefault( c => c.Key == key ).Child;
public List<Item> Array( string key ) =>
Arrays.TryGetValue( key, out var a ) ? a : new List<Item>();
}
/// <summary>An array member: either a plain value, a reference, or a nested element.</summary>
public sealed class Item
{
public string Value;
public Node Element;
public bool IsReference;
}
public static Node FindFirst( Node root, string type ) => FindAll( root, type ).FirstOrDefault();
public static List<Node> FindAll( Node root, string type )
{
var found = new List<Node>();
Walk( root );
return found;
void Walk( Node n )
{
if ( n is null )
return;
if ( n.Type == type )
found.Add( n );
foreach ( var (_, child) in n.Children )
Walk( child );
foreach ( var array in n.Arrays.Values )
{
foreach ( var item in array )
{
if ( item.Element is not null )
Walk( item.Element );
}
}
}
}
/// <summary>
/// Parse KeyValues2 text into a tree, or return null and say where it broke.
///
/// Strict about the two things the writer got wrong: array members must be separated by
/// commas, and a reference inside an element_array is the two tokens "element" "<id>".
/// </summary>
public static Node Parse( string text, out string error )
{
error = null;
try
{
var reader = new Reader( text );
reader.SkipHeader();
return reader.ReadElement();
}
catch ( FormatException e )
{
error = e.Message;
return null;
}
}
sealed class Reader
{
readonly string _text;
int _i;
public Reader( string text ) => _text = text;
int Line => _text.Take( _i ).Count( c => c == '\n' ) + 1;
public void SkipHeader()
{
SkipSpace();
if ( !_text.AsSpan( _i ).StartsWith( "<!--" ) )
throw new FormatException( "no <!-- dmx encoding --> header" );
var end = _text.IndexOf( "-->", _i, StringComparison.Ordinal );
if ( end < 0 )
throw new FormatException( "unterminated header comment" );
_i = end + 3;
}
/// <summary>An element: a quoted type name, then a braced body.</summary>
public Node ReadElement()
{
var node = new Node { Type = ReadQuoted() };
Expect( '{' );
while ( true )
{
SkipSpace();
if ( Peek() == '}' )
{
_i++;
break;
}
var key = ReadQuoted();
var type = ReadQuoted();
if ( type.EndsWith( "_array", StringComparison.Ordinal ) )
{
node.Arrays[key] = ReadArray( type );
continue;
}
// An attribute whose type is an element type name is an inline child element;
// anything else - including "element", which is a reference - is a plain value.
SkipSpace();
if ( Peek() == '{' )
{
var child = new Node { Type = type };
ReadBody( child );
node.Children.Add( (key, child) );
continue;
}
var value = ReadQuoted();
node.Values[key] = value;
if ( key == "name" )
node.Name = value;
}
return node;
}
void ReadBody( Node node )
{
Expect( '{' );
while ( true )
{
SkipSpace();
if ( Peek() == '}' )
{
_i++;
return;
}
var key = ReadQuoted();
var type = ReadQuoted();
if ( type.EndsWith( "_array", StringComparison.Ordinal ) )
{
node.Arrays[key] = ReadArray( type );
continue;
}
SkipSpace();
if ( Peek() == '{' )
{
var child = new Node { Type = type };
ReadBody( child );
node.Children.Add( (key, child) );
continue;
}
var value = ReadQuoted();
node.Values[key] = value;
if ( key == "name" )
node.Name = value;
}
}
List<Item> ReadArray( string arrayType )
{
Expect( '[' );
var items = new List<Item>();
var first = true;
while ( true )
{
SkipSpace();
if ( Peek() == ']' )
{
_i++;
return items;
}
// THE CHECK THIS FILE EXISTS FOR. Every member after the first has to be preceded
// by a comma, whether it is a scalar or a whole nested element.
if ( !first )
{
if ( Peek() != ',' )
throw new FormatException( $"line {Line}: expecting ',' between array members, found '{Peek()}'" );
_i++;
SkipSpace();
if ( Peek() == ']' )
throw new FormatException( $"line {Line}: trailing comma before ']'" );
}
first = false;
var token = ReadQuoted();
if ( arrayType != "element_array" )
{
items.Add( new Item { Value = token } );
continue;
}
// In an element_array a member is either "element" "<id>" - a reference - or a
// type name followed by a body.
if ( token == "element" )
{
items.Add( new Item { Value = ReadQuoted(), IsReference = true } );
continue;
}
SkipSpace();
if ( Peek() != '{' )
throw new FormatException(
$"line {Line}: '{token}' is read as an element type and needs a body; " +
"a bare id must be written as \"element\" \"<id>\"" );
var child = new Node { Type = token };
ReadBody( child );
items.Add( new Item { Element = child } );
}
}
string ReadQuoted()
{
SkipSpace();
if ( Peek() != '"' )
throw new FormatException( $"line {Line}: expecting a quoted token, found '{Peek()}'" );
_i++;
var sb = new System.Text.StringBuilder();
while ( _i < _text.Length && _text[_i] != '"' )
{
if ( _text[_i] == '\\' && _i + 1 < _text.Length )
_i++;
sb.Append( _text[_i++] );
}
if ( _i >= _text.Length )
throw new FormatException( $"line {Line}: unterminated string" );
_i++;
return sb.ToString();
}
void Expect( char c )
{
SkipSpace();
if ( Peek() != c )
throw new FormatException( $"line {Line}: expecting '{c}', found '{Peek()}'" );
_i++;
}
char Peek() => _i < _text.Length ? _text[_i] : '\0';
void SkipSpace()
{
while ( _i < _text.Length && char.IsWhiteSpace( _text[_i] ) )
_i++;
}
}
}