using Editor.Prism.Core;
using Editor.Prism.Text.LanguageDb;

namespace Editor.Prism.Text.Lexer;

/// <summary>
/// The bits packed into <see cref="LexState.Flags"/>. <see cref="LexState.Depth"/> carries the brace
/// nesting depth. Two lines that enter with the same flags and depth lex identically, which is what
/// makes the incremental re-lex driver's state-convergence check correct.
/// </summary>
public static class LexFlags
{
	/// <summary>Nothing carried over.</summary>
	public const int None = 0;

	/// <summary>The line begins inside a <c>/* … */</c> comment.</summary>
	public const int BlockComment = 1 << 0;

	/// <summary>The previous line was a preprocessor directive ending in a backslash.</summary>
	public const int PreprocessorContinuation = 1 << 1;

	/// <summary>The previous line ended inside a double-quoted string continued with a backslash.</summary>
	public const int StringContinuation = 1 << 2;

	/// <summary>The line begins inside a <c>&lt; … &gt;</c> metadata annotation.</summary>
	public const int Annotation = 1 << 3;

	/// <summary>Bit offset of the 4-bit current-VFX-block field.</summary>
	public const int BlockShift = 8;

	/// <summary>Mask of the current-VFX-block field.</summary>
	public const int BlockMask = 0xF << BlockShift;

	/// <summary>Bit offset of the 4-bit pending-VFX-block field, set between the name and its brace.</summary>
	public const int PendingBlockShift = 12;

	/// <summary>Mask of the pending-VFX-block field.</summary>
	public const int PendingBlockMask = 0xF << PendingBlockShift;

	/// <summary>Bit offset of the 4-bit attribute-bracket depth, so <c>[numthreads(\n…)]</c> works.</summary>
	public const int AttributeShift = 16;

	/// <summary>Mask of the attribute-bracket depth field.</summary>
	public const int AttributeMask = 0xF << AttributeShift;

	/// <summary>Deepest attribute nesting the flags word can record. Deeper nesting saturates here.</summary>
	public const int MaxAttributeDepth = 0xF;

	/// <summary>Bit offset of the 6-bit parenthesis depth, so a call split over lines stays a call.</summary>
	public const int ParenShift = 20;

	/// <summary>Mask of the parenthesis-depth field.</summary>
	public const int ParenMask = 0x3F << ParenShift;

	/// <summary>Deepest parenthesis nesting the flags word can record. Deeper nesting saturates here.</summary>
	public const int MaxParenDepth = 0x3F;

	/// <summary>Reads the 1-based VFX block index out of a flags word. Zero means file scope.</summary>
	public static int Block( int flags ) => ( flags & BlockMask ) >> BlockShift;

	/// <summary>Writes the 1-based VFX block index into a flags word.</summary>
	public static int WithBlock( int flags, int block ) =>
		( flags & ~BlockMask ) | ( ( block & 0xF ) << BlockShift );

	/// <summary>Reads the 1-based pending VFX block index out of a flags word.</summary>
	public static int PendingBlock( int flags ) => ( flags & PendingBlockMask ) >> PendingBlockShift;

	/// <summary>Writes the 1-based pending VFX block index into a flags word.</summary>
	public static int WithPendingBlock( int flags, int block ) =>
		( flags & ~PendingBlockMask ) | ( ( block & 0xF ) << PendingBlockShift );

	/// <summary>Reads the attribute-bracket depth out of a flags word.</summary>
	public static int Attribute( int flags ) => ( flags & AttributeMask ) >> AttributeShift;

	/// <summary>Writes the attribute-bracket depth into a flags word, saturating at the field width.</summary>
	public static int WithAttribute( int flags, int depth ) =>
		( flags & ~AttributeMask ) | ( Math.Clamp( depth, 0, MaxAttributeDepth ) << AttributeShift );

	/// <summary>Reads the parenthesis depth out of a flags word.</summary>
	public static int Paren( int flags ) => ( flags & ParenMask ) >> ParenShift;

	/// <summary>Writes the parenthesis depth into a flags word, saturating at the field width.</summary>
	public static int WithParen( int flags, int depth ) =>
		( flags & ~ParenMask ) | ( Math.Clamp( depth, 0, MaxParenDepth ) << ParenShift );
}

/// <summary>
/// Everything the identifier classifier knows about where an identifier sits, so a subclass can
/// refine the decision without re-implementing the scanner.
/// </summary>
/// <param name="AfterDot">The identifier follows a <c>.</c> — a member or a swizzle.</param>
/// <param name="AfterScope">The identifier follows a <c>::</c>.</param>
/// <param name="AfterColon">The identifier follows a single <c>:</c> — a semantic.</param>
/// <param name="BeforeParen">The next significant character is <c>(</c>.</param>
/// <param name="BeforeScope">The next significant characters are <c>::</c>.</param>
/// <param name="InAttribute">The identifier sits inside an attribute's <c>[ ]</c>.</param>
/// <param name="InAnnotation">The identifier sits inside a <c>&lt; … &gt;</c> metadata block.</param>
/// <param name="InDirective">The identifier sits on a preprocessor directive line.</param>
/// <param name="Depth">Brace nesting depth at this point.</param>
/// <param name="Flags">The live <see cref="LexFlags"/> word, carrying the VFX block context.</param>
/// <param name="PreviousWord">The previous identifier or keyword on this line, or null.</param>
public readonly record struct IdentifierContext(
	bool AfterDot,
	bool AfterScope,
	bool AfterColon,
	bool BeforeParen,
	bool BeforeScope,
	bool InAttribute,
	bool InAnnotation,
	bool InDirective,
	int Depth,
	int Flags,
	string PreviousWord );

/// <summary>
/// A resumable, line-based HLSL lexer. It never allocates beyond the substring it needs to classify
/// an identifier, never throws, and carries every scrap of cross-line state in <see cref="LexState"/>
/// so a document can be re-lexed from any line.
/// </summary>
public class HlslLexer : ILexer
{
	/// <inheritdoc/>
	public virtual string Language => "hlsl";

	/// <summary>The word tables and rules this lexer classifies against.</summary>
	protected virtual LanguageDefinition Definition => HlslLanguage.Definition;

	/// <inheritdoc/>
	public virtual bool IsInert( LexState state ) =>
		( state.Flags & ( LexFlags.BlockComment | LexFlags.StringContinuation ) ) != 0;

	/// <inheritdoc/>
	public LexState Lex( string line, LexState entry, List<Token> output )
	{
		if ( output is null )
			return entry;

		if ( line is null )
			return entry;

		try
		{
			return LexCore( line, entry, output );
		}
		catch ( Exception e )
		{
			// A lexer bug must never take the editor down or corrupt the state chain: colour the
			// line as an error, hand back the state we were given and carry on.
			PrismLog.Warn( $"Prism lexer failed on a line of {Language}: {e.Message}" );

			if ( line.Length > 0 )
				output.Add( new Token( 0, line.Length, TokenKind.Error ) );

			return entry;
		}
	}

	private LexState LexCore( string line, LexState entry, List<Token> output )
	{
		var def = Definition;
		var len = line.Length;
		var flags = entry.Flags;
		var depth = entry.Depth;
		var i = 0;

		var directive = ( flags & LexFlags.PreprocessorContinuation ) != 0;
		flags &= ~LexFlags.PreprocessorContinuation;

		var includePath = false;

		// Attribute and parenthesis nesting travel in the state word: `[numthreads(\n\t8, 8, 1 )]`
		// and a call split over several lines are both common, and a lexer that resets them at every
		// newline would classify the continuation lines as if they were at statement scope.
		var attributeDepth = LexFlags.Attribute( flags );
		var parenDepth = LexFlags.Paren( flags );

		var prevKind = TokenKind.None;
		var prevChar = '\0';
		var prevPrevKind = TokenKind.None;
		var prevPrevChar = '\0';
		var scopeBefore = false;
		string previousWord = null;
		string previousWord2 = null;

		// ---- resume a block comment ----
		if ( ( flags & LexFlags.BlockComment ) != 0 )
		{
			var end = line.IndexOf( "*/", StringComparison.Ordinal );

			if ( end < 0 )
			{
				Emit( output, 0, len, TokenKind.Comment );
				return new LexState( flags, depth );
			}

			Emit( output, 0, end + 2, TokenKind.Comment );
			flags &= ~LexFlags.BlockComment;
			i = end + 2;
		}
		// ---- resume a continued string ----
		else if ( ( flags & LexFlags.StringContinuation ) != 0 )
		{
			flags &= ~LexFlags.StringContinuation;
			i = ScanString( line, 0, '"', output, ref flags );
			prevKind = TokenKind.String;
			prevChar = '"';
		}

		while ( i < len )
		{
			var c = line[i];

			// ---- whitespace ----
			if ( c == ' ' || c == '\t' || c == '\r' || c == '\n' || char.IsWhiteSpace( c ) )
			{
				var ws = i;
				while ( i < len && char.IsWhiteSpace( line[i] ) )
					i++;

				Emit( output, ws, i - ws, TokenKind.Whitespace );
				continue;
			}

			// ---- comments ----
			if ( c == '/' && i + 1 < len && line[i + 1] == '/' )
			{
				var doc = i + 2 < len && ( line[i + 2] == '/' || line[i + 2] == '!' );
				Emit( output, i, len - i, doc ? TokenKind.DocComment : TokenKind.Comment );
				i = len;
				continue;
			}

			if ( c == '/' && i + 1 < len && line[i + 1] == '*' )
			{
				var end = line.IndexOf( "*/", i + 2, StringComparison.Ordinal );

				if ( end < 0 )
				{
					Emit( output, i, len - i, TokenKind.Comment );
					flags |= LexFlags.BlockComment;
					i = len;
					continue;
				}

				Emit( output, i, end + 2 - i, TokenKind.Comment );
				i = end + 2;
				continue;
			}

			// ---- preprocessor directive ----
			if ( c == '#' && !directive && prevKind == TokenKind.None )
			{
				var start = i;
				i++;

				while ( i < len && ( line[i] == ' ' || line[i] == '\t' ) )
					i++;

				var nameStart = i;
				while ( i < len && IsIdentifierPart( line[i] ) )
					i++;

				var name = line.Substring( nameStart, i - nameStart );
				Emit( output, start, i - start, TokenKind.Preprocessor );

				directive = true;
				includePath = name is "include" or "__include" or "import" or "implementing" or "module";

				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Preprocessor;
				prevChar = i > start ? line[i - 1] : '#';
				continue;
			}

			// ---- include path ----
			if ( includePath && ( c == '"' || c == '<' ) )
			{
				var close = c == '"' ? '"' : '>';
				var end = line.IndexOf( close, i + 1 );
				var stop = end < 0 ? len : end + 1;

				Emit( output, i, stop - i, TokenKind.IncludePath );
				i = stop;
				includePath = false;

				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.IncludePath;
				prevChar = close;
				continue;
			}

			// ---- strings and character literals ----
			if ( def.Comments.StringQuotes.IndexOf( c ) >= 0 )
			{
				i = ScanString( line, i, c, output, ref flags );

				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.String;
				prevChar = c;
				continue;
			}

			// ---- numbers ----
			if ( char.IsDigit( c ) || ( c == '.' && i + 1 < len && char.IsDigit( line[i + 1] ) ) )
			{
				var start = i;
				i = ScanNumber( line, i );
				Emit( output, start, i - start, TokenKind.Number );

				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Number;
				prevChar = line[i - 1];
				continue;
			}

			// ---- identifiers ----
			if ( IsIdentifierStart( c ) )
			{
				var start = i;
				while ( i < len && IsIdentifierPart( line[i] ) )
					i++;

				var word = line.Substring( start, i - start );
				var next = PeekSignificant( line, i, out var next2 );

				var ctx = new IdentifierContext(
					AfterDot: prevChar == '.' && !scopeBefore,
					AfterScope: scopeBefore,
					AfterColon: prevChar == ':' && prevKind == TokenKind.Operator && !scopeBefore,
					BeforeParen: next == '(',
					BeforeScope: next == ':' && next2 == ':',
					InAttribute: attributeDepth > 0,
					InAnnotation: ( flags & LexFlags.Annotation ) != 0,
					InDirective: directive,
					Depth: depth,
					Flags: flags,
					PreviousWord: previousWord );

				var kind = ClassifyIdentifier( word, ctx );
				Emit( output, start, i - start, kind );

				// `import foo;` / `module foo;` name their target with a bare identifier.
				includePath = false;
				previousWord2 = previousWord;
				previousWord = word;

				// A VFX block name at file scope arms the block that its next '{' opens.
				if ( kind == TokenKind.BlockKeyword )
				{
					var index = BlockIndex( word );
					if ( index > 0 )
						flags = LexFlags.WithPendingBlock( flags, index );
				}

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = kind;
				prevChar = line[i - 1];
				continue;
			}

			// ---- annotation open ----
			if ( c == '<' && def.SupportsAnnotations && ( flags & LexFlags.Annotation ) == 0 && !directive &&
				 parenDepth == 0 && attributeDepth == 0 &&
				 LooksLikeAnnotation( line, i, prevKind, prevChar, prevPrevKind, prevPrevChar, previousWord2 ) )
			{
				Emit( output, i, 1, TokenKind.Punctuation );
				flags |= LexFlags.Annotation;
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = '<';
				continue;
			}

			// ---- annotation close ----
			if ( c == '>' && ( flags & LexFlags.Annotation ) != 0 && parenDepth == 0 )
			{
				Emit( output, i, 1, TokenKind.Punctuation );
				flags &= ~LexFlags.Annotation;
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = '>';
				continue;
			}

			// ---- brackets, braces, parentheses ----
			if ( c == '[' )
			{
				if ( attributeDepth > 0 || IsAttributePosition( prevKind, prevChar ) )
					attributeDepth++;

				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = '[';
				continue;
			}

			if ( c == ']' )
			{
				if ( attributeDepth > 0 )
					attributeDepth--;

				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = ']';
				continue;
			}

			if ( c == '(' || c == ')' )
			{
				if ( c == '(' )
					parenDepth++;
				else if ( parenDepth > 0 )
					parenDepth--;

				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = c;
				continue;
			}

			if ( c == '{' )
			{
				depth++;

				// A brace can never sit inside an attribute or an argument list, so it is the one
				// unambiguous place to resynchronise both counters after malformed or macro-mangled text.
				attributeDepth = 0;
				parenDepth = 0;

				if ( depth == 1 )
				{
					var pending = LexFlags.PendingBlock( flags );
					if ( pending > 0 )
					{
						flags = LexFlags.WithBlock( flags, pending );
						flags = LexFlags.WithPendingBlock( flags, 0 );
					}
				}

				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = '{';
				continue;
			}

			if ( c == '}' )
			{
				if ( depth > 0 )
					depth--;

				attributeDepth = 0;
				parenDepth = 0;

				if ( depth == 0 )
				{
					flags = LexFlags.WithBlock( flags, 0 );
					flags = LexFlags.WithPendingBlock( flags, 0 );
				}

				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = '}';
				continue;
			}

			if ( c == ',' || c == ';' )
			{
				Emit( output, i, 1, TokenKind.Punctuation );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Punctuation;
				prevChar = c;
				continue;
			}

			// ---- line continuation ----
			if ( c == '\\' )
			{
				Emit( output, i, 1, directive ? TokenKind.Preprocessor : TokenKind.Operator );
				i++;

				scopeBefore = false;
				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Operator;
				prevChar = '\\';
				continue;
			}

			// ---- operators ----
			var op = MatchOperator( def, line, i );

			if ( op > 0 )
			{
				Emit( output, i, op, TokenKind.Operator );
				scopeBefore = op == 2 && line[i] == ':' && line[i + 1] == ':';

				prevPrevKind = prevKind;
				prevPrevChar = prevChar;
				prevKind = TokenKind.Operator;
				prevChar = line[i + op - 1];
				i += op;
				continue;
			}

			// ---- anything else ----
			Emit( output, i, 1, TokenKind.Error );
			i++;

			scopeBefore = false;
			prevPrevKind = prevKind;
			prevPrevChar = prevChar;
			prevKind = TokenKind.Error;
			prevChar = c;
		}

		// A directive continued with a trailing backslash keeps directive context alive.
		if ( directive && EndsWithBackslash( line ) )
			flags |= LexFlags.PreprocessorContinuation;

		// A directive is one logical line. Whatever it left unbalanced — `#define MIN(a,b)` bodies do
		// this constantly — must not leak into the code that follows it.
		if ( directive && ( flags & LexFlags.PreprocessorContinuation ) == 0 )
		{
			attributeDepth = 0;
			parenDepth = 0;
		}

		flags = LexFlags.WithAttribute( flags, attributeDepth );
		flags = LexFlags.WithParen( flags, parenDepth );

		return new LexState( flags, depth );
	}

	/// <summary>
	/// Decides what colour class an identifier belongs to. Subclasses override this to add
	/// language-specific words; everything else about the scan is shared.
	/// </summary>
	protected virtual TokenKind ClassifyIdentifier( string word, IdentifierContext ctx )
	{
		var def = Definition;

		// Members and swizzles.
		if ( ctx.AfterDot )
		{
			if ( ctx.BeforeParen && HlslLanguage.IsMemberMethod( word ) )
				return TokenKind.Intrinsic;

			return TokenKind.Identifier;
		}

		// Static members of an engine class: Material::Init, Depth::Get, Light::From.
		if ( ctx.AfterScope )
			return ctx.BeforeParen ? TokenKind.FunctionName : TokenKind.Identifier;

		// The name in `struct Foo`, `interface IBar`, `enum Baz` is being declared right here.
		if ( IsTypeIntroducer( ctx.PreviousWord ) && !def.IsKeyword( word ) && !def.IsModifier( word ) )
			return TokenKind.UserType;

		// Semantics.
		if ( ctx.AfterColon && def.IsSemantic( word ) )
			return TokenKind.Semantic;

		if ( word.Length > 3 && word[0] == 'S' && word[1] == 'V' && word[2] == '_' )
			return TokenKind.Semantic;

		// Attributes.
		if ( ctx.InAttribute )
			return def.IsAttribute( word ) ? TokenKind.Attribute : TokenKind.Identifier;

		// VFX block names at file scope.
		if ( def.HasVfxBlocks && ctx.Depth == 0 && SboxSymbols.IsBlockKeyword( word ) )
			return TokenKind.BlockKeyword;

		// Metadata annotations.
		if ( ctx.InAnnotation )
		{
			if ( def.HasSboxSymbols && SboxSymbols.IsComboSymbol( word ) )
				return TokenKind.Preprocessor;

			if ( def.IsLiteral( word ) )
				return TokenKind.Keyword;

			if ( def.IsType( word ) )
				return TokenKind.BuiltinType;

			return TokenKind.Annotation;
		}

		if ( def.IsControlKeyword( word ) )
			return TokenKind.ControlKeyword;

		if ( def.IsModifier( word ) )
			return TokenKind.Modifier;

		if ( def.IsLiteral( word ) )
			return TokenKind.Keyword;

		if ( def.IsType( word ) )
			return TokenKind.BuiltinType;

		if ( def.IsKeyword( word ) )
			return TokenKind.Keyword;

		if ( def.IsPredefinedMacro( word ) )
			return TokenKind.Preprocessor;

		if ( def.HasSboxSymbols && SboxSymbols.IsComboSymbol( word ) )
			return TokenKind.Preprocessor;

		if ( def.IsIntrinsic( word ) )
			return TokenKind.Intrinsic;

		if ( def.HasSboxSymbols && SboxSymbols.TryGet( word, out var symbol ) )
		{
			switch ( symbol.Kind )
			{
				case SboxSymbolKind.Global:
				case SboxSymbolKind.Constant:
					return TokenKind.EngineGlobal;

				case SboxSymbolKind.Macro:
				case SboxSymbolKind.Function:
				case SboxSymbolKind.Method:
					return TokenKind.Intrinsic;

				case SboxSymbolKind.Struct:
				case SboxSymbolKind.Enum:
					return TokenKind.UserType;

				case SboxSymbolKind.Annotation:
					return TokenKind.Annotation;

				case SboxSymbolKind.ComboDeclaration:
				case SboxSymbolKind.HeaderKey:
				case SboxSymbolKind.BlockKeyword:
					return TokenKind.Keyword;
			}
		}

		if ( def.HasSboxSymbols && word.Length > 2 && word[0] == 'g' && word[1] == '_' )
			return TokenKind.EngineGlobal;

		if ( ctx.BeforeScope )
			return TokenKind.UserType;

		if ( ctx.BeforeParen )
			return TokenKind.FunctionName;

		return TokenKind.Identifier;
	}

	/// <summary>True when the previous word introduces a type name that is being declared.</summary>
	protected static bool IsTypeIntroducer( string word ) =>
		word is "struct" or "class" or "interface" or "enum" or "extension" or "cbuffer" or "tbuffer" or "namespace";

	/// <summary>The 1-based index of a VFX block name, or zero when the word is not one.</summary>
	protected static int BlockIndex( string word )
	{
		var names = SboxSymbols.BlockNames;

		for ( var i = 0; i < names.Count; i++ )
		{
			if ( string.Equals( names[i], word, StringComparison.Ordinal ) )
				return i + 1;
		}

		return 0;
	}

	private static void Emit( List<Token> output, int start, int length, TokenKind kind )
	{
		if ( length > 0 )
			output.Add( new Token( start, length, kind ) );
	}

	private static bool IsIdentifierStart( char c ) => c == '_' || char.IsLetter( c );

	private static bool IsIdentifierPart( char c ) => c == '_' || char.IsLetterOrDigit( c );

	private static bool EndsWithBackslash( string line )
	{
		for ( var i = line.Length - 1; i >= 0; i-- )
		{
			var c = line[i];
			if ( char.IsWhiteSpace( c ) )
				continue;

			return c == '\\';
		}

		return false;
	}

	private static char PeekSignificant( string line, int from, out char second )
	{
		second = '\0';

		var i = from;
		while ( i < line.Length && char.IsWhiteSpace( line[i] ) )
			i++;

		if ( i >= line.Length )
			return '\0';

		if ( i + 1 < line.Length )
			second = line[i + 1];

		return line[i];
	}

	private static bool IsAttributePosition( TokenKind prevKind, char prevChar )
	{
		// `[branch]`, `[numthreads(8,8,1)]` and `[[vk::binding(0)]]` only ever follow the start of a
		// line or a statement boundary. `arr[i]` follows an identifier, a ')' or a ']'.
		if ( prevKind == TokenKind.None )
			return true;

		if ( prevKind != TokenKind.Punctuation && prevKind != TokenKind.Operator )
			return false;

		return prevChar is ';' or '{' or '}' or '[' or ':';
	}

	/// <summary>
	/// Whether a <c>&lt;</c> opens an s&amp;box metadata annotation rather than a comparison or a
	/// generic argument list.
	/// <para>
	/// Two independent tests have to agree. <b>To the left</b> the bracket must follow something that can
	/// end a declarator — <c>Texture2D g_tColor &lt;</c>, <c>float4 g_vTint &lt;</c>, or the closing
	/// parenthesis of <c>CreateTexture2D( g_t ) &lt;</c> and <c>… : register( t0 ) &lt;</c>, both of
	/// which appear in the shipped engine shaders. <b>To the right</b> the body must open the way the
	/// annotation grammar always opens: an identifier immediately followed by <c>(</c>. Every single-line
	/// annotation in the engine's ~170 shipped shaders and headers matches that shape, and no comparison
	/// or generic instantiation does — <c>a &lt; b</c>, <c>i &lt; count</c>, <c>Buffer&lt;float4&gt;</c>
	/// and <c>struct Foo&lt;T : IBar&gt;</c> are all rejected without needing the old "scan ahead for a
	/// terminator" heuristic, which got <c>x = f( a ) &lt; b;</c> wrong.
	/// </para>
	/// <para>
	/// The one shape that cannot be decided from the left of the newline is a bracket that ends its line.
	/// There the declarator test alone decides, with a generic parameter list ruled out by the word two
	/// back: <c>struct Foo&lt;</c> and <c>interface IBar&lt;</c> are generics, <c>Texture2D g_tRMA &lt;</c>
	/// is an annotation.
	/// </para>
	/// <para>
	/// <b>Residual shapes, measured against the 173 shipped engine shaders and headers and a suite of
	/// hand-written adversarial cases.</b> Two remain, both narrow and both harmless:
	/// </para>
	/// <list type="bullet">
	/// <item><description>
	/// A statement-scope comparison chain of the exact form <c>f( a ) &lt; g( b ) … ; … &gt;</c> — a
	/// closing parenthesis, a call, and a <c>;</c> immediately before the first same-level <c>&gt;</c>.
	/// Nothing shaped like that is valid HLSL, so no case is known; it is listed because it is what the
	/// three tests below cannot separate in principle. Note that the same chain inside a condition —
	/// <c>if ( Length( a ) &lt; Length( b ) || x &gt; y )</c> — never reaches here at all, because the
	/// caller only offers a <c>&lt;</c> at parenthesis depth zero.
	/// </description></item>
	/// <item><description>
	/// The mirror miss: an annotation that opens after a <c>)</c> <i>and</i> wraps onto the next line —
	/// <c>Texture2D g_t : register( t0 ) &lt;</c> with the body below it. It is read as a comparison and
	/// left uncoloured. No shipped shader writes one; deciding it would need the following line, which a
	/// resumable line lexer does not have.
	/// </description></item>
	/// </list>
	/// </summary>
	private static bool LooksLikeAnnotation( string line, int lt, TokenKind prevKind, char prevChar,
		TokenKind prevPrevKind, char prevPrevChar, string previousWord2 )
	{
		if ( !EndsDeclarator( prevKind, prevChar, prevPrevKind, prevPrevChar, previousWord2 ) )
			return false;

		// A bracket after `)` is the weaker of the two left-hand shapes — `bool b = Foo( a ) < Bar( c );`
		// wears it too — so that form has to close on its own line, and close the way the annotation
		// grammar closes: with the `;` that ends the last entry sitting immediately before the `>`. All
		// twelve `) < … >` annotations in the shipped engine shaders end `; >`; a comparison chain such
		// as `Foo( a ) < Bar( b ) && c > d;` has its `;` on the far side of the `>` and is rejected.
		var afterParen = prevKind == TokenKind.Punctuation;

		if ( afterParen && !ClosesLikeAnnotation( line, lt ) )
			return false;

		var i = lt + 1;

		while ( i < line.Length && char.IsWhiteSpace( line[i] ) )
			i++;

		// `Texture2D g_tRMA <` with the body on the following lines, or a trailing comment.
		if ( i >= line.Length || ( line[i] == '/' && i + 1 < line.Length && ( line[i + 1] == '/' || line[i + 1] == '*' ) ) )
			return !afterParen;

		if ( !IsIdentifierStart( line[i] ) )
			return false;

		while ( i < line.Length && IsIdentifierPart( line[i] ) )
			i++;

		while ( i < line.Length && char.IsWhiteSpace( line[i] ) )
			i++;

		return i < line.Length && line[i] == '(';
	}

	/// <summary>
	/// True when the run starting at <paramref name="lt"/> closes on this line the way an annotation
	/// body closes: a <c>&gt;</c> at the same parenthesis and bracket depth as the opening
	/// <c>&lt;</c>, with the <c>;</c> that terminates the last entry immediately before it.
	/// </summary>
	private static bool ClosesLikeAnnotation( string line, int lt )
	{
		var depth = 0;

		for ( var i = lt + 1; i < line.Length; i++ )
		{
			var c = line[i];

			if ( c == '(' || c == '[' )
			{
				depth++;
				continue;
			}

			if ( c == ')' || c == ']' )
			{
				// An unbalanced closer means the `<` was never the head of a bracketed run at all.
				if ( depth == 0 )
					return false;

				depth--;
				continue;
			}

			if ( c != '>' || depth != 0 )
				continue;

			var j = i - 1;

			while ( j > lt && char.IsWhiteSpace( line[j] ) )
				j--;

			return j > lt && line[j] == ';';
		}

		return false;
	}

	/// <summary>True when the token before a <c>&lt;</c> could be the end of a declarator.</summary>
	private static bool EndsDeclarator( TokenKind prevKind, char prevChar, TokenKind prevPrevKind,
		char prevPrevChar, string previousWord2 )
	{
		// `CreateTexture2D( g_tFoo ) < … >;` and `Texture2D g_t : register( t0 ) < … >;`.
		if ( prevKind == TokenKind.Punctuation )
			return prevChar == ')';

		if ( prevKind != TokenKind.Identifier && prevKind != TokenKind.EngineGlobal &&
			 prevKind != TokenKind.UserType && prevKind != TokenKind.FunctionName )
			return false;

		// `struct Foo<T>` / `interface IBar<T>` declare a generic, never an annotation.
		if ( IsTypeIntroducer( previousWord2 ) )
			return false;

		return prevPrevKind == TokenKind.BuiltinType ||
			   prevPrevKind == TokenKind.UserType ||
			   prevPrevKind == TokenKind.Identifier ||
			   prevPrevKind == TokenKind.EngineGlobal ||
			   prevPrevKind == TokenKind.Keyword ||
			   prevPrevKind == TokenKind.Modifier ||
			   ( prevPrevKind == TokenKind.Operator && prevPrevChar == '>' );
	}

	private static int MatchOperator( LanguageDefinition def, string line, int i )
	{
		var operators = def.Operators;

		for ( var o = 0; o < operators.Count; o++ )
		{
			var op = operators[o];
			if ( op.Length == 0 || i + op.Length > line.Length )
				continue;

			var match = true;
			for ( var k = 0; k < op.Length; k++ )
			{
				if ( line[i + k] != op[k] )
				{
					match = false;
					break;
				}
			}

			if ( match )
				return op.Length;
		}

		return 0;
	}

	private static int ScanNumber( string line, int start )
	{
		var len = line.Length;
		var i = start;

		if ( line[i] == '0' && i + 1 < len && ( line[i + 1] == 'x' || line[i + 1] == 'X' ) )
		{
			i += 2;
			while ( i < len && ( Uri.IsHexDigit( line[i] ) || line[i] == '_' ) )
				i++;
		}
		else
		{
			var seenDot = false;

			while ( i < len )
			{
				var c = line[i];

				if ( char.IsDigit( c ) || c == '_' )
				{
					i++;
					continue;
				}

				if ( c == '.' && !seenDot )
				{
					// `0..1` is a combo range, not a malformed float: stop before the second dot.
					if ( i + 1 < len && line[i + 1] == '.' )
						break;

					seenDot = true;
					i++;
					continue;
				}

				if ( ( c == 'e' || c == 'E' ) && i + 1 < len &&
					 ( char.IsDigit( line[i + 1] ) || ( ( line[i + 1] == '+' || line[i + 1] == '-' ) && i + 2 < len && char.IsDigit( line[i + 2] ) ) ) )
				{
					i += 2;
					continue;
				}

				break;
			}
		}

		// Literal suffixes: f F h H u U l L, and the 16/32/64-bit spellings.
		while ( i < len && ( line[i] is 'f' or 'F' or 'h' or 'H' or 'u' or 'U' or 'l' or 'L' ) )
			i++;

		return i;
	}

	private static int ScanString( string line, int start, char quote, List<Token> output, ref int flags )
	{
		var len = line.Length;
		var i = start < len && line[start] == quote ? start + 1 : start;
		var closed = false;

		while ( i < len )
		{
			var c = line[i];

			if ( c == '\\' && i + 1 < len )
			{
				i += 2;
				continue;
			}

			if ( c == quote )
			{
				i++;
				closed = true;
				break;
			}

			i++;
		}

		Emit( output, start, i - start, TokenKind.String );

		if ( !closed && quote == '"' && i >= len && len > 0 && line[len - 1] == '\\' )
			flags |= LexFlags.StringContinuation;

		return i;
	}
}