Framework/Lua/Runtime/Compiler/Lexer.cs

Tokenizer for the framework's Lua 5.1 interpreter, with the GLua extensions (!=, &&, ||, // and /* */ comments, continue). Turns source text into a list of Token values with line numbers and handles strings, long strings, numeric literals and syntax errors.

using System.Collections.Generic;
using System.Text;

namespace RoleplayFramework.Lua.Compiler;

internal enum TokenKind
{
	Eof,
	Name,
	Number,
	String,

	// keywords
	And, Break, Continue, Do, Else, ElseIf, End, False, For, Function, If, In, Local, Nil, Not, Or, Repeat, Return, Then, True, Until, While,

	// symbols
	Plus, Minus, Star, Slash, Percent, Caret, Hash,
	Eq, NotEq, LessEq, GreaterEq, Less, Greater, Assign,
	LParen, RParen, LBrace, RBrace, LBracket, RBracket,
	Semicolon, Colon, Comma, Dot, Concat, Ellipsis,
}

internal readonly struct Token
{
	public readonly TokenKind Kind;
	public readonly string Text;
	public readonly double Number;
	public readonly int Line;

	public Token( TokenKind kind, int line, string text = null, double number = 0 )
	{
		Kind = kind;
		Line = line;
		Text = text;
		Number = number;
	}

	public override string ToString() => Kind switch
	{
		TokenKind.Name => Text,
		TokenKind.String => $"'{Text}'",
		TokenKind.Number => LuaValue.NumberToString( Number ),
		TokenKind.Eof => "<eof>",
		_ => Lexer.Describe( Kind ),
	};
}

/// <summary>
/// Lua 5.1 lexer with the common Garry's Mod (GLua) extensions:
/// <c>!=</c>, <c>!</c>, <c>&amp;&amp;</c>, <c>||</c>, <c>//</c> and <c>/* */</c> comments, and <c>continue</c>.
/// </summary>
internal sealed class Lexer
{
	readonly string _src;
	readonly string _chunk;
	int _pos;
	int _line = 1;

	static readonly Dictionary<string, TokenKind> Keywords = new()
	{
		["and"] = TokenKind.And, ["break"] = TokenKind.Break, ["continue"] = TokenKind.Continue,
		["do"] = TokenKind.Do, ["else"] = TokenKind.Else, ["elseif"] = TokenKind.ElseIf, ["end"] = TokenKind.End,
		["false"] = TokenKind.False, ["for"] = TokenKind.For, ["function"] = TokenKind.Function, ["if"] = TokenKind.If,
		["in"] = TokenKind.In, ["local"] = TokenKind.Local, ["nil"] = TokenKind.Nil, ["not"] = TokenKind.Not,
		["or"] = TokenKind.Or, ["repeat"] = TokenKind.Repeat, ["return"] = TokenKind.Return, ["then"] = TokenKind.Then,
		["true"] = TokenKind.True, ["until"] = TokenKind.Until, ["while"] = TokenKind.While,
	};

	public Lexer( string source, string chunkName )
	{
		_src = source ?? "";
		_chunk = chunkName;
		// Skip a shebang line, like the reference implementation.
		if ( _src.StartsWith( "#" ) )
		{
			while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
		}
	}

	public static string Describe( TokenKind k ) => k switch
	{
		TokenKind.Plus => "+", TokenKind.Minus => "-", TokenKind.Star => "*", TokenKind.Slash => "/",
		TokenKind.Percent => "%", TokenKind.Caret => "^", TokenKind.Hash => "#", TokenKind.Eq => "==",
		TokenKind.NotEq => "~=", TokenKind.LessEq => "<=", TokenKind.GreaterEq => ">=", TokenKind.Less => "<",
		TokenKind.Greater => ">", TokenKind.Assign => "=", TokenKind.LParen => "(", TokenKind.RParen => ")",
		TokenKind.LBrace => "{", TokenKind.RBrace => "}", TokenKind.LBracket => "[", TokenKind.RBracket => "]",
		TokenKind.Semicolon => ";", TokenKind.Colon => ":", TokenKind.Comma => ",", TokenKind.Dot => ".",
		TokenKind.Concat => "..", TokenKind.Ellipsis => "...", TokenKind.Eof => "<eof>",
		_ => k.ToString().ToLowerInvariant(),
	};

	LuaSyntaxException Error( string msg ) => new( _chunk, _line, msg );

	char Peek( int offset = 0 ) => _pos + offset < _src.Length ? _src[_pos + offset] : '\0';

	public List<Token> Tokenize()
	{
		var list = new List<Token>();
		while ( true )
		{
			var t = NextToken();
			list.Add( t );
			if ( t.Kind == TokenKind.Eof ) return list;
		}
	}

	Token NextToken()
	{
		SkipWhitespaceAndComments();
		if ( _pos >= _src.Length ) return new Token( TokenKind.Eof, _line );

		char c = _src[_pos];
		int line = _line;

		if ( char.IsAsciiLetter( c ) || c == '_' )
		{
			int start = _pos;
			while ( _pos < _src.Length && (char.IsAsciiLetterOrDigit( _src[_pos] ) || _src[_pos] == '_') ) _pos++;
			var word = _src.Substring( start, _pos - start );
			return Keywords.TryGetValue( word, out var kw ) ? new Token( kw, line, word ) : new Token( TokenKind.Name, line, word );
		}

		if ( char.IsAsciiDigit( c ) || (c == '.' && char.IsAsciiDigit( Peek( 1 ) )) )
			return ReadNumber();

		switch ( c )
		{
			case '"':
			case '\'':
				return new Token( TokenKind.String, line, ReadQuotedString( c ) );
			case '[':
				{
					int level = LongBracketLevel();
					if ( level >= 0 ) return new Token( TokenKind.String, line, ReadLongString( level ) );
					_pos++;
					return new Token( TokenKind.LBracket, line );
				}
			case '+': _pos++; return new Token( TokenKind.Plus, line );
			case '-': _pos++; return new Token( TokenKind.Minus, line );
			case '*': _pos++; return new Token( TokenKind.Star, line );
			case '/': _pos++; return new Token( TokenKind.Slash, line );
			case '%': _pos++; return new Token( TokenKind.Percent, line );
			case '^': _pos++; return new Token( TokenKind.Caret, line );
			case '#': _pos++; return new Token( TokenKind.Hash, line );
			case '(': _pos++; return new Token( TokenKind.LParen, line );
			case ')': _pos++; return new Token( TokenKind.RParen, line );
			case '{': _pos++; return new Token( TokenKind.LBrace, line );
			case '}': _pos++; return new Token( TokenKind.RBrace, line );
			case ']': _pos++; return new Token( TokenKind.RBracket, line );
			case ';': _pos++; return new Token( TokenKind.Semicolon, line );
			case ',': _pos++; return new Token( TokenKind.Comma, line );
			case ':': _pos++; return new Token( TokenKind.Colon, line );
			case '=':
				_pos++;
				if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.Eq, line ); }
				return new Token( TokenKind.Assign, line );
			case '<':
				_pos++;
				if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.LessEq, line ); }
				return new Token( TokenKind.Less, line );
			case '>':
				_pos++;
				if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.GreaterEq, line ); }
				return new Token( TokenKind.Greater, line );
			case '~':
				if ( Peek( 1 ) == '=' ) { _pos += 2; return new Token( TokenKind.NotEq, line ); }
				throw Error( "unexpected symbol near '~'" );
			case '!':
				if ( Peek( 1 ) == '=' ) { _pos += 2; return new Token( TokenKind.NotEq, line ); }
				_pos++;
				return new Token( TokenKind.Not, line, "!" );
			case '&':
				if ( Peek( 1 ) == '&' ) { _pos += 2; return new Token( TokenKind.And, line, "&&" ); }
				throw Error( "unexpected symbol near '&'" );
			case '|':
				if ( Peek( 1 ) == '|' ) { _pos += 2; return new Token( TokenKind.Or, line, "||" ); }
				throw Error( "unexpected symbol near '|'" );
			case '.':
				if ( Peek( 1 ) == '.' )
				{
					if ( Peek( 2 ) == '.' ) { _pos += 3; return new Token( TokenKind.Ellipsis, line ); }
					_pos += 2;
					return new Token( TokenKind.Concat, line );
				}
				_pos++;
				return new Token( TokenKind.Dot, line );
		}

		throw Error( $"unexpected symbol near '{c}'" );
	}

	void SkipWhitespaceAndComments()
	{
		while ( _pos < _src.Length )
		{
			char c = _src[_pos];
			if ( c == '\n' ) { _line++; _pos++; continue; }
			if ( c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v' ) { _pos++; continue; }

			if ( c == '-' && Peek( 1 ) == '-' )
			{
				_pos += 2;
				if ( Peek() == '[' )
				{
					int level = LongBracketLevel();
					if ( level >= 0 ) { ReadLongString( level ); continue; }
				}
				while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
				continue;
			}

			// GLua comment styles
			if ( c == '/' && Peek( 1 ) == '/' )
			{
				while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
				continue;
			}
			if ( c == '/' && Peek( 1 ) == '*' )
			{
				int startLine = _line;
				_pos += 2;
				while ( true )
				{
					if ( _pos >= _src.Length ) throw new LuaSyntaxException( _chunk, startLine, "unfinished block comment" );
					if ( _src[_pos] == '*' && Peek( 1 ) == '/' ) { _pos += 2; break; }
					if ( _src[_pos] == '\n' ) _line++;
					_pos++;
				}
				continue;
			}
			break;
		}
	}

	// At '[': returns the level of a long bracket ([[ = 0, [=[ = 1 ...) or -1 if this isn't one.
	int LongBracketLevel()
	{
		int p = _pos + 1;
		int level = 0;
		while ( p < _src.Length && _src[p] == '=' ) { level++; p++; }
		if ( p < _src.Length && _src[p] == '[' ) return level;
		return -1;
	}

	string ReadLongString( int level )
	{
		int startLine = _line;
		_pos += level + 2;
		// A newline directly after the opening bracket is skipped.
		if ( Peek() == '\r' ) _pos++;
		if ( Peek() == '\n' ) { _pos++; _line++; }

		var sb = new StringBuilder();
		while ( true )
		{
			if ( _pos >= _src.Length ) throw new LuaSyntaxException( _chunk, startLine, "unfinished long string/comment" );
			char c = _src[_pos];
			if ( c == ']' )
			{
				int p = _pos + 1;
				int l = 0;
				while ( p < _src.Length && _src[p] == '=' ) { l++; p++; }
				if ( l == level && p < _src.Length && _src[p] == ']' )
				{
					_pos = p + 1;
					return sb.ToString();
				}
			}
			if ( c == '\n' ) _line++;
			if ( c != '\r' ) sb.Append( c );
			_pos++;
		}
	}

	string ReadQuotedString( char quote )
	{
		_pos++;
		var sb = new StringBuilder();
		while ( true )
		{
			if ( _pos >= _src.Length ) throw Error( "unfinished string" );
			char c = _src[_pos];
			if ( c == quote ) { _pos++; return sb.ToString(); }
			if ( c == '\n' ) throw Error( "unfinished string" );

			if ( c != '\\' ) { sb.Append( c ); _pos++; continue; }

			_pos++;
			char e = Peek();
			switch ( e )
			{
				case 'n': sb.Append( '\n' ); _pos++; break;
				case 't': sb.Append( '\t' ); _pos++; break;
				case 'r': sb.Append( '\r' ); _pos++; break;
				case 'a': sb.Append( '\a' ); _pos++; break;
				case 'b': sb.Append( '\b' ); _pos++; break;
				case 'f': sb.Append( '\f' ); _pos++; break;
				case 'v': sb.Append( '\v' ); _pos++; break;
				case '\\': sb.Append( '\\' ); _pos++; break;
				case '"': sb.Append( '"' ); _pos++; break;
				case '\'': sb.Append( '\'' ); _pos++; break;
				case '\n': sb.Append( '\n' ); _line++; _pos++; break;
				case 'x':
					{
						int h1 = LuaValue.HexDigit( Peek( 1 ) ), h2 = LuaValue.HexDigit( Peek( 2 ) );
						if ( h1 < 0 || h2 < 0 ) throw Error( "hexadecimal digit expected" );
						sb.Append( (char)(h1 * 16 + h2) );
						_pos += 3;
						break;
					}
				case 'z':
					_pos++;
					while ( _pos < _src.Length && char.IsWhiteSpace( _src[_pos] ) )
					{
						if ( _src[_pos] == '\n' ) _line++;
						_pos++;
					}
					break;
				default:
					if ( char.IsAsciiDigit( e ) )
					{
						int v = 0, n = 0;
						while ( n < 3 && char.IsAsciiDigit( Peek() ) ) { v = v * 10 + (Peek() - '0'); _pos++; n++; }
						if ( v > 255 ) throw Error( "escape sequence too large" );
						sb.Append( (char)v );
						break;
					}
					throw Error( $"invalid escape sequence '\\{e}'" );
			}
		}
	}

	Token ReadNumber()
	{
		int line = _line;
		int start = _pos;

		if ( Peek() == '0' && (Peek( 1 ) == 'x' || Peek( 1 ) == 'X') )
		{
			_pos += 2;
			while ( LuaValue.HexDigit( Peek() ) >= 0 ) _pos++;
		}
		else
		{
			while ( char.IsAsciiDigit( Peek() ) || Peek() == '.' ) _pos++;
			if ( Peek() == 'e' || Peek() == 'E' )
			{
				_pos++;
				if ( Peek() == '+' || Peek() == '-' ) _pos++;
				while ( char.IsAsciiDigit( Peek() ) ) _pos++;
			}
		}

		// A number immediately followed by a name character is malformed (e.g. "3x").
		while ( char.IsAsciiLetterOrDigit( Peek() ) || Peek() == '_' ) _pos++;

		var text = _src.Substring( start, _pos - start );
		if ( !LuaValue.TryParseNumber( text, out var value ) )
			throw Error( $"malformed number near '{text}'" );

		return new Token( TokenKind.Number, line, text, value );
	}
}