SboxSuperuser/Core/Effects/EffectLexer.cs

Lexer for a small effects scripting language. Splits source into token records (comments, numbers, text, hex colors, placeholders, keywords, operators) and records diagnostics instead of throwing on unclosed comments/text or malformed #colors. Also has a helper to unquote text tokens.

namespace SboxSuperuser.Core.Effects;

/// <summary>
/// Splits a script into tokens. It never throws: unexpected characters become Invalid tokens with a diagnostic,
/// so the editor can still color everything around them.
/// </summary>
internal static class EffectLexer
{
	internal static readonly string[] Keywords = { "if", "else", "true", "false", "and", "or", "not" };

	// Longest operators first so "==" is not read as two "=".
	private static readonly string[] Operators =
	{
		"==", "!=", "<=", ">=", "&&", "||", "+", "-", "*", "/", "%", "<", ">", "=", "!", "?", ":", "(", ")", "[", "]", "{", "}", ",", ";"
	};

	internal static IReadOnlyList<EffectToken> Tokenize( string source, List<EffectDiagnostic> diagnostics )
	{
		var tokens = new List<EffectToken>();
		var index = 0;
		while ( index < source.Length )
		{
			var c = source[ index ];
			if ( char.IsWhiteSpace( c ) )
			{
				index++;
				continue;
			}

			var start = index;
			if ( c == '/' && Peek( source, index + 1 ) == '/' )
			{
				while ( index < source.Length && source[ index ] != '\n' ) index++;
				tokens.Add( new EffectToken( EffectTokenKind.Comment, source[ start..index ], start, index - start ) );
			}
			else if ( c == '/' && Peek( source, index + 1 ) == '*' )
			{
				var close = source.IndexOf( "*/", index + 2, StringComparison.Ordinal );
				index = close < 0 ? source.Length : close + 2;
				if ( close < 0 ) diagnostics.Add( new EffectDiagnostic( "this comment is never closed; end it with */", start, 2 ) );
				tokens.Add( new EffectToken( EffectTokenKind.Comment, source[ start..index ], start, index - start ) );
			}
			else if ( char.IsAsciiDigit( c ) || c == '.' && char.IsAsciiDigit( Peek( source, index + 1 ) ) )
			{
				while ( index < source.Length && (char.IsAsciiDigit( source[ index ] ) || source[ index ] == '.') ) index++;
				tokens.Add( new EffectToken( EffectTokenKind.Number, source[ start..index ], start, index - start ) );
			}
			else if ( c == '"' )
			{
				index = ReadText( source, index, diagnostics, tokens );
			}
			else if ( c == '#' )
			{
				index++;
				while ( index < source.Length && char.IsAsciiHexDigit( source[ index ] ) ) index++;
				var text = source[ start..index ];
				var valid = EffectColor.TryParseHex( text, out _ );
				if ( !valid ) diagnostics.Add( new EffectDiagnostic( $"\"{text}\" is not a color; use #rgb, #rrggbb or #rrggbbaa", start, index - start ) );
				tokens.Add( new EffectToken( valid ? EffectTokenKind.Color : EffectTokenKind.Invalid, text, start, index - start ) );
			}
			else if ( c == '%' && IsPlaceholder( source, index, out var end ) )
			{
				index = end;
				tokens.Add( new EffectToken( EffectTokenKind.Placeholder, source[ start..index ], start, index - start ) );
			}
			else if ( char.IsLetter( c ) || c == '_' )
			{
				while ( index < source.Length && (char.IsLetterOrDigit( source[ index ] ) || source[ index ] == '_') ) index++;
				var word = source[ start..index ];
				tokens.Add( new EffectToken( Keywords.Contains( word ) ? EffectTokenKind.Keyword : EffectTokenKind.Identifier, word, start, index - start ) );
			}
			else if ( Operators.FirstOrDefault( op => string.CompareOrdinal( source, index, op, 0, op.Length ) == 0 ) is { } op )
			{
				index += op.Length;
				tokens.Add( new EffectToken( EffectTokenKind.Operator, op, start, op.Length ) );
			}
			else
			{
				index++;
				diagnostics.Add( new EffectDiagnostic( $"\"{c}\" cannot be used here", start, 1 ) );
				tokens.Add( new EffectToken( EffectTokenKind.Invalid, c.ToString(), start, 1 ) );
			}
		}

		tokens.Add( new EffectToken( EffectTokenKind.End, "", source.Length, 0 ) );
		return tokens;
	}

	private static int ReadText( string source, int start, List<EffectDiagnostic> diagnostics, List<EffectToken> tokens )
	{
		var index = start + 1;
		while ( index < source.Length && source[ index ] != '"' && source[ index ] != '\n' )
			index += source[ index ] == '\\' && index + 1 < source.Length ? 2 : 1;
		var closed = index < source.Length && source[ index ] == '"';
		if ( closed ) index++;
		else diagnostics.Add( new EffectDiagnostic( "this text is never closed; end it with \"", start, index - start ) );
		tokens.Add( new EffectToken( EffectTokenKind.Text, source[ start..index ], start, index - start ) );
		return index;
	}

	// A placeholder is %word% with no spaces, so "a % b" stays the remainder operator.
	private static bool IsPlaceholder( string source, int start, out int end )
	{
		end = start + 1;
		while ( end < source.Length && char.IsAsciiLetter( source[ end ] ) ) end++;
		if ( end == start + 1 || end >= source.Length || source[ end ] != '%' ) return false;
		end++;
		return true;
	}

	private static char Peek( string source, int index ) => index < source.Length ? source[ index ] : '\0';

	/// <summary>The value of a text token with its quotes removed and \" \\ \n escapes applied.</summary>
	internal static string Unquote( string token )
	{
		var body = token.Length >= 2 && token[ ^1 ] == '"' ? token[ 1..^1 ] : token[ 1.. ];
		return body.Replace( "\\n", "\n" ).Replace( "\\\"", "\"" ).Replace( "\\\\", "\\" );
	}
}