Editor-side KV3 parser and tokenizer for .vmdl source. Tokenizes KV3 text including comments and strings, builds token stream, and parses into a tree of Kv3Node types (Kv3Object, Kv3Array, Kv3Scalar) with helper Kv3Document utilities for finding objects by _class.
using System;
using System.Text;
namespace ModelPro.Vmdl;
internal enum Kv3TokenType
{
LBrace,
RBrace,
LBracket,
RBracket,
Equals,
Comma,
String,
Ident,
End
}
internal readonly struct Kv3Token
{
public readonly Kv3TokenType Type;
public readonly int Start;
public readonly int End;
public readonly string Text;
public Kv3Token( Kv3TokenType type, int start, int end, string text )
{
Type = type;
Start = start;
End = end;
Text = text;
}
}
/// <summary>
/// Tokenizes KV3 text (the format model .vmdl source files are stored in).
/// Handles the <!-- ... --> header comment, // and /* */ comments, quoted strings
/// with escapes, and barewords (identifiers and numbers).
/// </summary>
internal sealed class Kv3Tokenizer
{
readonly string src;
int pos;
public Kv3Tokenizer( string src )
{
this.src = src;
}
public List<Kv3Token> Tokenize()
{
var tokens = new List<Kv3Token>();
while ( true )
{
SkipTrivia();
if ( pos >= src.Length )
{
tokens.Add( new Kv3Token( Kv3TokenType.End, pos, pos, "" ) );
break;
}
int start = pos;
char c = src[pos];
switch ( c )
{
case '{':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.LBrace, start, pos, "{" ) );
break;
case '}':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.RBrace, start, pos, "}" ) );
break;
case '[':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.LBracket, start, pos, "[" ) );
break;
case ']':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.RBracket, start, pos, "]" ) );
break;
case '=':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.Equals, start, pos, "=" ) );
break;
case ',':
pos++;
tokens.Add( new Kv3Token( Kv3TokenType.Comma, start, pos, "," ) );
break;
case '"':
ReadString( tokens, start );
break;
default:
ReadBareword( tokens, start );
break;
}
}
return tokens;
}
void ReadString( List<Kv3Token> tokens, int start )
{
pos++; // opening quote
while ( pos < src.Length && src[pos] != '"' )
{
if ( src[pos] == '\\' && pos + 1 < src.Length )
pos += 2;
else
pos++;
}
if ( pos < src.Length )
pos++; // closing quote
tokens.Add( new Kv3Token( Kv3TokenType.String, start, pos, src.Substring( start, pos - start ) ) );
}
void ReadBareword( List<Kv3Token> tokens, int start )
{
while ( pos < src.Length )
{
char c = src[pos];
if ( char.IsWhiteSpace( c ) || c is '{' or '}' or '[' or ']' or '=' or ',' )
break;
if ( c == '/' && pos + 1 < src.Length && src[pos + 1] is '/' or '*' )
break;
if ( c == '<' && pos + 3 < src.Length && src.AsSpan( pos ).StartsWith( "<!--" ) )
break;
pos++;
}
tokens.Add( new Kv3Token( Kv3TokenType.Ident, start, pos, src.Substring( start, pos - start ) ) );
}
void SkipTrivia()
{
while ( pos < src.Length )
{
char c = src[pos];
if ( char.IsWhiteSpace( c ) )
{
pos++;
continue;
}
if ( c == '/' && pos + 1 < src.Length && src[pos + 1] == '/' )
{
pos += 2;
while ( pos < src.Length && src[pos] != '\n' )
pos++;
continue;
}
if ( c == '/' && pos + 1 < src.Length && src[pos + 1] == '*' )
{
var end = src.IndexOf( "*/", pos + 2, StringComparison.Ordinal );
pos = end < 0 ? src.Length : end + 2;
continue;
}
if ( c == '<' && pos + 3 < src.Length && src.AsSpan( pos ).StartsWith( "<!--" ) )
{
var end = src.IndexOf( "-->", pos, StringComparison.Ordinal );
pos = end < 0 ? src.Length : end + 3;
continue;
}
break;
}
}
}
/// <summary>A parsed node in a KV3 document.</summary>
public abstract class Kv3Node
{
public int Start;
public int End;
}
/// <summary>An object - a list of key/value fields.</summary>
public sealed class Kv3Object : Kv3Node
{
public List<Kv3Field> Fields = new();
public Kv3Field FindField( string key ) => Fields.FirstOrDefault( f => f.Key == key );
/// <summary>The value of the "_class" field, used to identify node types in modeldoc files.</summary>
public string Class => (FindField( "_class" )?.Value as Kv3Scalar)?.Value;
}
public sealed class Kv3Field
{
public string Key;
public int KeyStart;
public int KeyEnd;
public Kv3Node Value;
}
/// <summary>An array of values.</summary>
public sealed class Kv3Array : Kv3Node
{
public List<Kv3Node> Items = new();
}
/// <summary>A scalar value - a string, number, or bareword.</summary>
public sealed class Kv3Scalar : Kv3Node
{
public bool IsString;
public string Raw;
public string Value;
}
/// <summary>
/// A parsed KV3 text document. Keeps a reference to the original source so edits
/// can be applied surgically without reformatting the whole file.
/// </summary>
public sealed class Kv3Document
{
public string Source { get; }
public Kv3Object Root { get; }
Kv3Document( string source )
{
Source = source;
Root = Kv3Parser.Parse( source );
}
public static Kv3Document Parse( string source ) => new( source );
/// <summary>
/// Walk the tree and return every object whose "_class" field matches the given name.
/// </summary>
public List<Kv3Object> FindObjects( string className )
{
var result = new List<Kv3Object>();
Walk( Root, result, className );
return result;
}
static void Walk( Kv3Node node, List<Kv3Object> result, string className )
{
if ( node is Kv3Object obj )
{
if ( obj.Class == className )
result.Add( obj );
foreach ( var f in obj.Fields )
Walk( f.Value, result, className );
}
else if ( node is Kv3Array arr )
{
foreach ( var item in arr.Items )
Walk( item, result, className );
}
}
}
internal sealed class Kv3Parser
{
readonly string src;
readonly List<Kv3Token> tokens;
int index;
Kv3Parser( string src, List<Kv3Token> tokens )
{
this.src = src;
this.tokens = tokens;
}
public static Kv3Object Parse( string source )
{
var tokens = new Kv3Tokenizer( source ).Tokenize();
var parser = new Kv3Parser( source, tokens );
return parser.ParseObject();
}
Kv3Token Peek => tokens[index];
Kv3Token Next() => tokens[index++];
bool AtEnd => Peek.Type == Kv3TokenType.End;
Kv3Node ParseValue()
{
var tok = Peek;
if ( tok.Type == Kv3TokenType.LBrace )
return ParseObject();
if ( tok.Type == Kv3TokenType.LBracket )
return ParseArray();
return ParseScalar();
}
Kv3Object ParseObject()
{
var open = Next(); // {
var obj = new Kv3Object { Start = open.Start };
while ( true )
{
if ( AtEnd )
break;
if ( Peek.Type == Kv3TokenType.RBrace )
{
var close = Next();
obj.End = close.End;
break;
}
if ( Peek.Type == Kv3TokenType.Comma )
{
Next();
continue;
}
var keyTok = Peek;
if ( keyTok.Type is not (Kv3TokenType.String or Kv3TokenType.Ident) )
{
Next();
continue;
}
Next(); // consume key
string key = keyTok.Type == Kv3TokenType.String
? Unquote( keyTok.Text )
: keyTok.Text;
if ( Peek.Type == Kv3TokenType.Equals )
Next();
var value = ParseValue();
obj.Fields.Add( new Kv3Field { Key = key, KeyStart = keyTok.Start, KeyEnd = keyTok.End, Value = value } );
}
return obj;
}
Kv3Array ParseArray()
{
var open = Next(); // [
var arr = new Kv3Array { Start = open.Start };
while ( true )
{
if ( AtEnd )
break;
if ( Peek.Type == Kv3TokenType.RBracket )
{
var close = Next();
arr.End = close.End;
break;
}
if ( Peek.Type == Kv3TokenType.Comma )
{
Next();
continue;
}
arr.Items.Add( ParseValue() );
}
return arr;
}
Kv3Scalar ParseScalar()
{
var tok = Next();
return new Kv3Scalar
{
Start = tok.Start,
End = tok.End,
IsString = tok.Type == Kv3TokenType.String,
Raw = tok.Text,
Value = tok.Type == Kv3TokenType.String ? Unquote( tok.Text ) : tok.Text
};
}
static string Unquote( string s )
{
if ( s.Length < 2 || s[0] != '"' )
return s;
var inner = s.Substring( 1, s.Length - 2 );
var sb = new StringBuilder( inner.Length );
for ( int i = 0; i < inner.Length; i++ )
{
if ( inner[i] == '\\' && i + 1 < inner.Length )
{
char n = inner[++i];
switch ( n )
{
case 'n':
sb.Append( '\n' );
break;
case 't':
sb.Append( '\t' );
break;
case 'r':
sb.Append( '\r' );
break;
case '"':
sb.Append( '"' );
break;
case '\'':
sb.Append( '\'' );
break;
case '\\':
sb.Append( '\\' );
break;
default:
sb.Append( n );
break;
}
}
else
{
sb.Append( inner[i] );
}
}
return sb.ToString();
}
}