Tokenizer for the framework's Lua 5.1 interpreter, with the GLua extensions (!=, &&, ||, // and /* */ comments, continue). Turns source text into a list of Token values with line numbers and handles strings, long strings, numeric literals and syntax errors.
using System.Collections.Generic;
using System.Text;
namespace RoleplayFramework.Lua.Compiler;
internal enum TokenKind
{
Eof,
Name,
Number,
String,
// keywords
And, Break, Continue, Do, Else, ElseIf, End, False, For, Function, If, In, Local, Nil, Not, Or, Repeat, Return, Then, True, Until, While,
// symbols
Plus, Minus, Star, Slash, Percent, Caret, Hash,
Eq, NotEq, LessEq, GreaterEq, Less, Greater, Assign,
LParen, RParen, LBrace, RBrace, LBracket, RBracket,
Semicolon, Colon, Comma, Dot, Concat, Ellipsis,
}
internal readonly struct Token
{
public readonly TokenKind Kind;
public readonly string Text;
public readonly double Number;
public readonly int Line;
public Token( TokenKind kind, int line, string text = null, double number = 0 )
{
Kind = kind;
Line = line;
Text = text;
Number = number;
}
public override string ToString() => Kind switch
{
TokenKind.Name => Text,
TokenKind.String => $"'{Text}'",
TokenKind.Number => LuaValue.NumberToString( Number ),
TokenKind.Eof => "<eof>",
_ => Lexer.Describe( Kind ),
};
}
/// <summary>
/// Lua 5.1 lexer with the common Garry's Mod (GLua) extensions:
/// <c>!=</c>, <c>!</c>, <c>&&</c>, <c>||</c>, <c>//</c> and <c>/* */</c> comments, and <c>continue</c>.
/// </summary>
internal sealed class Lexer
{
readonly string _src;
readonly string _chunk;
int _pos;
int _line = 1;
static readonly Dictionary<string, TokenKind> Keywords = new()
{
["and"] = TokenKind.And, ["break"] = TokenKind.Break, ["continue"] = TokenKind.Continue,
["do"] = TokenKind.Do, ["else"] = TokenKind.Else, ["elseif"] = TokenKind.ElseIf, ["end"] = TokenKind.End,
["false"] = TokenKind.False, ["for"] = TokenKind.For, ["function"] = TokenKind.Function, ["if"] = TokenKind.If,
["in"] = TokenKind.In, ["local"] = TokenKind.Local, ["nil"] = TokenKind.Nil, ["not"] = TokenKind.Not,
["or"] = TokenKind.Or, ["repeat"] = TokenKind.Repeat, ["return"] = TokenKind.Return, ["then"] = TokenKind.Then,
["true"] = TokenKind.True, ["until"] = TokenKind.Until, ["while"] = TokenKind.While,
};
public Lexer( string source, string chunkName )
{
_src = source ?? "";
_chunk = chunkName;
// Skip a shebang line, like the reference implementation.
if ( _src.StartsWith( "#" ) )
{
while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
}
}
public static string Describe( TokenKind k ) => k switch
{
TokenKind.Plus => "+", TokenKind.Minus => "-", TokenKind.Star => "*", TokenKind.Slash => "/",
TokenKind.Percent => "%", TokenKind.Caret => "^", TokenKind.Hash => "#", TokenKind.Eq => "==",
TokenKind.NotEq => "~=", TokenKind.LessEq => "<=", TokenKind.GreaterEq => ">=", TokenKind.Less => "<",
TokenKind.Greater => ">", TokenKind.Assign => "=", TokenKind.LParen => "(", TokenKind.RParen => ")",
TokenKind.LBrace => "{", TokenKind.RBrace => "}", TokenKind.LBracket => "[", TokenKind.RBracket => "]",
TokenKind.Semicolon => ";", TokenKind.Colon => ":", TokenKind.Comma => ",", TokenKind.Dot => ".",
TokenKind.Concat => "..", TokenKind.Ellipsis => "...", TokenKind.Eof => "<eof>",
_ => k.ToString().ToLowerInvariant(),
};
LuaSyntaxException Error( string msg ) => new( _chunk, _line, msg );
char Peek( int offset = 0 ) => _pos + offset < _src.Length ? _src[_pos + offset] : '\0';
public List<Token> Tokenize()
{
var list = new List<Token>();
while ( true )
{
var t = NextToken();
list.Add( t );
if ( t.Kind == TokenKind.Eof ) return list;
}
}
Token NextToken()
{
SkipWhitespaceAndComments();
if ( _pos >= _src.Length ) return new Token( TokenKind.Eof, _line );
char c = _src[_pos];
int line = _line;
if ( char.IsAsciiLetter( c ) || c == '_' )
{
int start = _pos;
while ( _pos < _src.Length && (char.IsAsciiLetterOrDigit( _src[_pos] ) || _src[_pos] == '_') ) _pos++;
var word = _src.Substring( start, _pos - start );
return Keywords.TryGetValue( word, out var kw ) ? new Token( kw, line, word ) : new Token( TokenKind.Name, line, word );
}
if ( char.IsAsciiDigit( c ) || (c == '.' && char.IsAsciiDigit( Peek( 1 ) )) )
return ReadNumber();
switch ( c )
{
case '"':
case '\'':
return new Token( TokenKind.String, line, ReadQuotedString( c ) );
case '[':
{
int level = LongBracketLevel();
if ( level >= 0 ) return new Token( TokenKind.String, line, ReadLongString( level ) );
_pos++;
return new Token( TokenKind.LBracket, line );
}
case '+': _pos++; return new Token( TokenKind.Plus, line );
case '-': _pos++; return new Token( TokenKind.Minus, line );
case '*': _pos++; return new Token( TokenKind.Star, line );
case '/': _pos++; return new Token( TokenKind.Slash, line );
case '%': _pos++; return new Token( TokenKind.Percent, line );
case '^': _pos++; return new Token( TokenKind.Caret, line );
case '#': _pos++; return new Token( TokenKind.Hash, line );
case '(': _pos++; return new Token( TokenKind.LParen, line );
case ')': _pos++; return new Token( TokenKind.RParen, line );
case '{': _pos++; return new Token( TokenKind.LBrace, line );
case '}': _pos++; return new Token( TokenKind.RBrace, line );
case ']': _pos++; return new Token( TokenKind.RBracket, line );
case ';': _pos++; return new Token( TokenKind.Semicolon, line );
case ',': _pos++; return new Token( TokenKind.Comma, line );
case ':': _pos++; return new Token( TokenKind.Colon, line );
case '=':
_pos++;
if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.Eq, line ); }
return new Token( TokenKind.Assign, line );
case '<':
_pos++;
if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.LessEq, line ); }
return new Token( TokenKind.Less, line );
case '>':
_pos++;
if ( Peek() == '=' ) { _pos++; return new Token( TokenKind.GreaterEq, line ); }
return new Token( TokenKind.Greater, line );
case '~':
if ( Peek( 1 ) == '=' ) { _pos += 2; return new Token( TokenKind.NotEq, line ); }
throw Error( "unexpected symbol near '~'" );
case '!':
if ( Peek( 1 ) == '=' ) { _pos += 2; return new Token( TokenKind.NotEq, line ); }
_pos++;
return new Token( TokenKind.Not, line, "!" );
case '&':
if ( Peek( 1 ) == '&' ) { _pos += 2; return new Token( TokenKind.And, line, "&&" ); }
throw Error( "unexpected symbol near '&'" );
case '|':
if ( Peek( 1 ) == '|' ) { _pos += 2; return new Token( TokenKind.Or, line, "||" ); }
throw Error( "unexpected symbol near '|'" );
case '.':
if ( Peek( 1 ) == '.' )
{
if ( Peek( 2 ) == '.' ) { _pos += 3; return new Token( TokenKind.Ellipsis, line ); }
_pos += 2;
return new Token( TokenKind.Concat, line );
}
_pos++;
return new Token( TokenKind.Dot, line );
}
throw Error( $"unexpected symbol near '{c}'" );
}
void SkipWhitespaceAndComments()
{
while ( _pos < _src.Length )
{
char c = _src[_pos];
if ( c == '\n' ) { _line++; _pos++; continue; }
if ( c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v' ) { _pos++; continue; }
if ( c == '-' && Peek( 1 ) == '-' )
{
_pos += 2;
if ( Peek() == '[' )
{
int level = LongBracketLevel();
if ( level >= 0 ) { ReadLongString( level ); continue; }
}
while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
continue;
}
// GLua comment styles
if ( c == '/' && Peek( 1 ) == '/' )
{
while ( _pos < _src.Length && _src[_pos] != '\n' ) _pos++;
continue;
}
if ( c == '/' && Peek( 1 ) == '*' )
{
int startLine = _line;
_pos += 2;
while ( true )
{
if ( _pos >= _src.Length ) throw new LuaSyntaxException( _chunk, startLine, "unfinished block comment" );
if ( _src[_pos] == '*' && Peek( 1 ) == '/' ) { _pos += 2; break; }
if ( _src[_pos] == '\n' ) _line++;
_pos++;
}
continue;
}
break;
}
}
// At '[': returns the level of a long bracket ([[ = 0, [=[ = 1 ...) or -1 if this isn't one.
int LongBracketLevel()
{
int p = _pos + 1;
int level = 0;
while ( p < _src.Length && _src[p] == '=' ) { level++; p++; }
if ( p < _src.Length && _src[p] == '[' ) return level;
return -1;
}
string ReadLongString( int level )
{
int startLine = _line;
_pos += level + 2;
// A newline directly after the opening bracket is skipped.
if ( Peek() == '\r' ) _pos++;
if ( Peek() == '\n' ) { _pos++; _line++; }
var sb = new StringBuilder();
while ( true )
{
if ( _pos >= _src.Length ) throw new LuaSyntaxException( _chunk, startLine, "unfinished long string/comment" );
char c = _src[_pos];
if ( c == ']' )
{
int p = _pos + 1;
int l = 0;
while ( p < _src.Length && _src[p] == '=' ) { l++; p++; }
if ( l == level && p < _src.Length && _src[p] == ']' )
{
_pos = p + 1;
return sb.ToString();
}
}
if ( c == '\n' ) _line++;
if ( c != '\r' ) sb.Append( c );
_pos++;
}
}
string ReadQuotedString( char quote )
{
_pos++;
var sb = new StringBuilder();
while ( true )
{
if ( _pos >= _src.Length ) throw Error( "unfinished string" );
char c = _src[_pos];
if ( c == quote ) { _pos++; return sb.ToString(); }
if ( c == '\n' ) throw Error( "unfinished string" );
if ( c != '\\' ) { sb.Append( c ); _pos++; continue; }
_pos++;
char e = Peek();
switch ( e )
{
case 'n': sb.Append( '\n' ); _pos++; break;
case 't': sb.Append( '\t' ); _pos++; break;
case 'r': sb.Append( '\r' ); _pos++; break;
case 'a': sb.Append( '\a' ); _pos++; break;
case 'b': sb.Append( '\b' ); _pos++; break;
case 'f': sb.Append( '\f' ); _pos++; break;
case 'v': sb.Append( '\v' ); _pos++; break;
case '\\': sb.Append( '\\' ); _pos++; break;
case '"': sb.Append( '"' ); _pos++; break;
case '\'': sb.Append( '\'' ); _pos++; break;
case '\n': sb.Append( '\n' ); _line++; _pos++; break;
case 'x':
{
int h1 = LuaValue.HexDigit( Peek( 1 ) ), h2 = LuaValue.HexDigit( Peek( 2 ) );
if ( h1 < 0 || h2 < 0 ) throw Error( "hexadecimal digit expected" );
sb.Append( (char)(h1 * 16 + h2) );
_pos += 3;
break;
}
case 'z':
_pos++;
while ( _pos < _src.Length && char.IsWhiteSpace( _src[_pos] ) )
{
if ( _src[_pos] == '\n' ) _line++;
_pos++;
}
break;
default:
if ( char.IsAsciiDigit( e ) )
{
int v = 0, n = 0;
while ( n < 3 && char.IsAsciiDigit( Peek() ) ) { v = v * 10 + (Peek() - '0'); _pos++; n++; }
if ( v > 255 ) throw Error( "escape sequence too large" );
sb.Append( (char)v );
break;
}
throw Error( $"invalid escape sequence '\\{e}'" );
}
}
}
Token ReadNumber()
{
int line = _line;
int start = _pos;
if ( Peek() == '0' && (Peek( 1 ) == 'x' || Peek( 1 ) == 'X') )
{
_pos += 2;
while ( LuaValue.HexDigit( Peek() ) >= 0 ) _pos++;
}
else
{
while ( char.IsAsciiDigit( Peek() ) || Peek() == '.' ) _pos++;
if ( Peek() == 'e' || Peek() == 'E' )
{
_pos++;
if ( Peek() == '+' || Peek() == '-' ) _pos++;
while ( char.IsAsciiDigit( Peek() ) ) _pos++;
}
}
// A number immediately followed by a name character is malformed (e.g. "3x").
while ( char.IsAsciiLetterOrDigit( Peek() ) || Peek() == '_' ) _pos++;
var text = _src.Substring( start, _pos - start );
if ( !LuaValue.TryParseNumber( text, out var value ) )
throw Error( $"malformed number near '{text}'" );
return new Token( TokenKind.Number, line, text, value );
}
}