Editor/Prism/Compiler/Backends/VfxBlockValidator.cs

Validator for Prism VFX block IR modules. It checks MODES entries, combo declarations and values, global parameter and attribute collisions, varyings, stages and struct/helper naming and reports diagnostics to a sink.

Reflection
using Editor.Prism.Compiler.Ir;
using Editor.Prism.Core;

namespace Editor.Prism.Compiler.Backends;

/// <summary>
/// Pre-flight validation of everything the engine's native VFX front-end parses before a single line
/// of HLSL is compiled: block headers, MODES entries, feature and combo declarations, and the
/// declaration namespace.
/// <para>
/// This exists because those failures are invisible. When <c>HEADER</c>, <c>MODES</c>,
/// <c>FEATURES</c> or a combo fails to parse, the managed compile returns
/// <c>Success == false</c> with an <em>empty</em> program list and the real message goes only to the
/// native log. Catching it here turns a mystery into a sentence.
/// </para>
/// </summary>
public static class VfxBlockValidator
{
	/// <summary>How a validation pass reports a hard error.</summary>
	delegate void ErrorReporter( string code, string message, string detail = null );

	/// <summary>Every render pass the engine's mode parser accepts.</summary>
	public static readonly IReadOnlyList<string> KnownModes =
	[
		SboxShaderTemplates.ModeForward,
		SboxShaderTemplates.ModeVrForward,
		SboxShaderTemplates.ModeDefault,
		SboxShaderTemplates.ModeDepth,
		SboxShaderTemplates.ModeToolsShadingComplexity,
		SboxShaderTemplates.ModeToolsUtil
	];

	/// <summary>
	/// Modes the engine used to have and now hard-errors on, with the reason. <c>ToolsVis</c> became a
	/// runtime attribute; <c>common/classes/ToolsVis.hlsl</c> <c>#error</c>s if the combo is defined.
	/// </summary>
	public static readonly IReadOnlyDictionary<string, string> RemovedModes = new Dictionary<string, string>
	{
		["ToolsVis"] = "ToolsVis() was removed; tools visualisation is driven by the ToolsVisMode render attribute and ShadingModelStandard::Shade handles it.",
		["ToolsWireframe"] = "ToolsWireframe() was removed; wireframe is driven by the g_bWireframeMode render attribute."
	};

	/// <summary>
	/// Features the shipped <c>common/features.hlsl</c> already declares. Declaring one again is the
	/// native error <em>"Found feature %s multiple times"</em>.
	/// </summary>
	public static readonly IReadOnlyList<string> StockFeatures =
	[
		"F_DO_NOT_CAST_SHADOWS", "F_RENDER_BACKFACES", "F_MORPH_SUPPORTED",
		"F_TEXTURE_FILTERING", "F_ADDITIVE_BLEND"
	];

	/// <summary>
	/// Combos the shipped headers already declare for you. Re-declaring one is the native error
	/// <em>"Found combo %s multiple times declared differently"</em>.
	/// </summary>
	public static readonly IReadOnlyList<string> StockCombos =
	[
		"S_MODE_DEPTH", "D_BAKED_LIGHTING_FROM_LIGHTMAP", "D_BLENDMODE", "D_OPAQUE_FADE",
		"D_CS_VERTEX_ANIMATION", "D_COMPRESSED_NORMALS_AND_TANGENTS"
	];

	/// <summary>
	/// Parameters the engine's own headers declare. Redeclaring one is a duplicate-symbol compile
	/// error deep inside an include the user never wrote.
	/// </summary>
	public static readonly IReadOnlyList<string> ReservedNames =
	[
		"g_flAlphaTestReference", "g_flAntiAliasedEdgeStrength", "g_flOpacityScale",
		"g_bFogEnabled", "g_bWireframeMode", "g_vWireframeColor",
		"g_bWrinkleOverride", "g_flWrinkleOverride",
		"g_tDepthChain", "g_SheetTexture", "g_tBlueNoise", "BRDFLookup",
		"ToolsVisMode", "FlatOverlayColor", "ShaderIDColor", "ShadingComplexity",
		"g_flTime", "g_vCameraPositionWs", "g_vCameraDirWs", "g_flNearPlane", "g_flFarPlane",
		"g_vViewportSize", "g_vInvViewportSize", "g_vViewportOffset", "g_vRenderTargetSize",
		"g_vHighPrecisionLightingOffsetWs", "g_DirectionalLightColor", "g_DirectionalLightDirection",
		"g_sDefault", "g_sAniso", "g_sBilinearClamp", "g_sBilinearWrap", "g_sBilinearMirror",
		"g_sTrilinearWrap", "g_sTrilinearClamp", "g_sTrilinearMirror", "g_sTrilinearBorder",
		"g_sPointBorder", "g_sPointClamp", "g_sPointWrap", "g_sPointMirror"
	];

	/// <summary>The mandatory name prefix for a combo of a given kind.</summary>
	public static string ComboPrefix( ComboKind kind ) => kind switch
	{
		ComboKind.Feature => "F_",
		ComboKind.Static => "S_",
		_ => "D_"
	};

	/// <summary>True when a combo name carries the prefix its kind requires.</summary>
	public static bool HasValidPrefix( string name, ComboKind kind ) =>
		!string.IsNullOrEmpty( name ) && name.StartsWith( ComboPrefix( kind ), StringComparison.Ordinal );

	/// <summary>True when a mode name is one the engine's mode parser recognises.</summary>
	public static bool IsKnownMode( string mode )
	{
		var name = NormalizeMode( mode );

		foreach ( var known in KnownModes )
		{
			if ( string.Equals( known, name, StringComparison.Ordinal ) ) return true;
		}

		return false;
	}

	/// <summary>True when the name belongs to the engine and must not be redeclared.</summary>
	public static bool IsReserved( string name )
	{
		foreach ( var reserved in ReservedNames )
		{
			if ( string.Equals( reserved, name, StringComparison.Ordinal ) ) return true;
		}

		return false;
	}

	/// <summary>True when a string is a legal HLSL identifier.</summary>
	/// <remarks>
	/// ASCII only, deliberately: the engine's block grammar is an ANTLR parser over ASCII and a single
	/// accented letter fails the whole file with a mismatched-token dump and no line number.
	/// <c>char.IsLetter</c> is Unicode-aware and would call that name legal.
	/// </remarks>
	public static bool IsValidIdentifier( string name ) => SboxShaderTemplates.IsAsciiIdentifier( name );

	/// <summary>Strip a mode's call syntax, leaving the bare name.</summary>
	public static string NormalizeMode( string mode )
	{
		if ( string.IsNullOrWhiteSpace( mode ) ) return string.Empty;

		var name = mode.Trim();
		var paren = name.IndexOf( '(' );

		if ( paren >= 0 ) name = name[..paren];

		return name.Trim().TrimEnd( ';' ).Trim();
	}

	/// <summary>Validate a module and collect the problems, without a sink.</summary>
	public static IReadOnlyList<Diagnostic> Validate( IrModule module )
	{
		var sink = new DiagnosticSink();

		Validate( module, sink );

		return sink.All;
	}

	/// <summary>
	/// Validate a module into a sink. Returns false when at least one error was reported, which is the
	/// caller's cue not to bother writing a file the engine cannot parse.
	/// </summary>
	public static bool Validate( IrModule module, DiagnosticSink diagnostics )
	{
		diagnostics ??= new DiagnosticSink();

		if ( module?.Meta is null )
		{
			diagnostics.Error( DiagnosticCode.InvalidBlock, "There is no module to validate." );
			return false;
		}

		var errors = 0;

		void Error( string code, string message, string detail = null )
		{
			errors++;
			diagnostics.Error( code, message, null, detail );
		}

		ValidateModes( module, Error, diagnostics );
		ValidateCombos( module, Error, diagnostics );
		ValidateGlobals( module, Error, diagnostics );
		ValidateVaryings( module, Error, diagnostics );
		ValidateStructure( module, Error, diagnostics );

		return errors == 0;
	}

	static void ValidateModes( IrModule module, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var meta = module.Meta;

		// An empty list is not an error: the writer falls back to the domain's standard pass set, and
		// validating that same set here keeps the two from ever disagreeing.
		var modes = meta.Modes.Count > 0
			? (IReadOnlyList<string>)meta.Modes
			: SboxShaderTemplates.DefaultModesFor( meta.Domain );

		if ( modes.Count == 0 )
		{
			error( DiagnosticCode.InvalidBlock,
				"The shader declares no render passes.",
				"A MODES block with at least one entry is required; a surface shader normally declares Forward, Depth and ToolsShadingComplexity." );
			return;
		}

		var seen = new HashSet<string>( StringComparer.Ordinal );

		foreach ( var mode in modes )
		{
			var name = NormalizeMode( mode );

			if ( name.Length == 0 )
			{
				error( DiagnosticCode.InvalidBlock, "A MODES entry is empty." );
				continue;
			}

			if ( RemovedModes.TryGetValue( name, out var reason ) )
			{
				error( DiagnosticCode.InvalidBlock, $"'{name}' is no longer a valid render pass.", reason );
				continue;
			}

			if ( !IsKnownMode( name ) )
			{
				error( DiagnosticCode.InvalidBlock,
					$"'{name}' is not a render pass the engine recognises.",
					$"Known passes: {string.Join( ", ", KnownModes )}." );
				continue;
			}

			if ( !seen.Add( name ) )
			{
				diagnostics.Warn( DiagnosticCode.InvalidBlock, $"Render pass '{name}' is declared more than once." );
			}
		}
	}

	static void ValidateCombos( IrModule module, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var seen = new HashSet<string>( StringComparer.Ordinal );

		foreach ( var combo in module.Meta.Combos )
		{
			if ( combo is null ) continue;

			var name = combo.Name;
			var prefix = ComboPrefix( combo.Kind );

			if ( string.IsNullOrWhiteSpace( name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"A {combo.Kind} combo has no name.", null );
				continue;
			}

			if ( !IsValidIdentifier( name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"'{name}' is not a legal combo identifier.",
					"Combo names may contain only unaccented letters, digits and underscores, and cannot start " +
					"with a digit. A combo name is written straight into the FEATURES block and into every #if " +
					"that reads it, so unlike a parameter it cannot be renamed on the way out." );
				continue;
			}

			if ( !HasValidPrefix( name, combo.Kind ) )
			{
				error( DiagnosticCode.InvalidBlock,
					$"A {combo.Kind.ToString().ToLowerInvariant()} combo must start with \"{prefix}\", but '{name}' does not.",
					$"The engine aborts the whole shader compile with \"A {( combo.Kind == ComboKind.Dynamic ? "dynamic" : "static" )} combo doesn't start with \\\"{prefix}\\\"!\"." );
				continue;
			}

			if ( name != name.ToUpperInvariant() )
			{
				diagnostics.Warn( DiagnosticCode.InvalidBlock,
					$"Combo '{name}' is not upper case.", null,
					"Every combo in the engine and in every shipped shader is upper case; mixed case works but reads as a bug." );
			}

			if ( !seen.Add( name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"Combo '{name}' is declared more than once.",
					"The engine reports \"Found combo %s multiple times declared differently\" and aborts." );
				continue;
			}

			if ( combo.Kind == ComboKind.Feature && StockFeatures.Contains( name ) )
			{
				error( DiagnosticCode.InvalidBlock,
					$"Feature '{name}' is already declared by common/features.hlsl.",
					"Remove the keyword and use the stock feature, or rename yours." );
				continue;
			}

			if ( combo.Kind != ComboKind.Feature && StockCombos.Contains( name ) )
			{
				error( DiagnosticCode.InvalidBlock,
					$"Combo '{name}' is already declared by a shipped header.",
					"Re-declaring it produces \"Found combo %s multiple times declared differently\"." );
				continue;
			}

			ValidateComboValues( combo, error, diagnostics );
		}
	}

	static void ValidateComboValues( ComboDecl combo, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var values = combo.Values ?? Array.Empty<string>();
		var count = values.Count;

		if ( count == 1 )
		{
			error( DiagnosticCode.InvalidBlock,
				$"Combo '{combo.Name}' declares a single value.",
				"A combo is a range 0..N with N >= 1. Give it at least two values, or none at all for a plain 0..1 checkbox." );
			return;
		}

		if ( count > 0 )
		{
			var labelled = 0;

			for ( int i = 0; i < count; i++ )
			{
				var value = values[i];

				if ( string.IsNullOrWhiteSpace( value ) )
				{
					error( DiagnosticCode.InvalidBlock,
						$"Combo '{combo.Name}' value {i} has no label.",
						"The engine reports \"is missing an option string\" and aborts. Every value in a labelled range needs one." );
					continue;
				}

				var separator = value.IndexOf( '=' );

				if ( separator < 0 ) continue;

				labelled++;

				if ( !int.TryParse( value[..separator].Trim(), out var index ) )
				{
					error( DiagnosticCode.InvalidBlock,
						$"Combo '{combo.Name}' value '{value}' is not in the N=\"Label\" form.",
						"The engine requires \"N=\\\"String\\\" where N = combo value\"." );
					continue;
				}

				if ( index != i )
				{
					error( DiagnosticCode.InvalidBlock,
						$"Combo '{combo.Name}' value '{value}' is out of order: it sits at position {i} but claims index {index}.",
						"Value lists must be contiguous and ascending from 0; the engine reports \"has a string index out of order\"." );
				}
			}

			if ( labelled > 0 && labelled != count )
			{
				error( DiagnosticCode.InvalidBlock,
					$"Combo '{combo.Name}' mixes explicitly indexed values with unindexed ones.",
					"Either every value carries its index or none of them do." );
			}
		}

		var maximum = Math.Max( 1, count - 1 );

		if ( combo.Default < 0 || combo.Default > maximum )
		{
			error( DiagnosticCode.InvalidBlock,
				$"Combo '{combo.Name}' defaults to {combo.Default}, which is outside its range 0..{maximum}.", null );
		}
	}

	static void ValidateGlobals( IrModule module, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var byName = new Dictionary<string, GlobalDecl>( StringComparer.Ordinal );
		var byAttribute = new Dictionary<string, string>( StringComparer.Ordinal );
		var samplers = 0;

		foreach ( var global in module.Globals )
		{
			if ( global is null ) continue;

			// The spelling the backend will actually emit, not the raw one: a name outside the ASCII range
			// is renamed on the way out rather than rejected, so the user is never stopped from calling a
			// parameter 'Größe'. Anything the rename cannot repair is still an error.
			if ( !IsValidIdentifier( SboxShaderTemplates.SafeIdentifier( global.Name ) ) )
			{
				error( DiagnosticCode.GlobalCollision,
					$"'{global.Name}' is not a legal declaration name.",
					"Parameter names become HLSL identifiers: letters, digits and underscores only, never starting with a digit." );
				continue;
			}

			if ( IsReserved( global.Name ) )
			{
				error( DiagnosticCode.GlobalCollision,
					$"'{global.Name}' is declared by the engine's own headers and must not be redeclared.",
					"Rename the parameter. The engine's declaration arrives through common/shared.hlsl or common/pixel.hlsl and will collide." );
				continue;
			}

			if ( byName.TryGetValue( global.Name, out var existing ) )
			{
				if ( existing.ConflictsWith( global ) )
				{
					error( DiagnosticCode.GlobalCollision,
						$"Two different declarations are both named '{global.Name}'.",
						$"'{existing}' and '{global}'. Declarations are deduplicated by name, so a name can only mean one thing." );
				}
				else
				{
					diagnostics.Warn( DiagnosticCode.GlobalCollision,
						$"'{global.Name}' is declared more than once. The duplicate is ignored." );
				}

				continue;
			}

			byName[global.Name] = global;

			if ( global.Kind == GlobalKind.Sampler ) samplers++;

			if ( string.IsNullOrEmpty( global.AttributeName ) ) continue;

			if ( byAttribute.TryGetValue( global.AttributeName, out var owner ) )
			{
				error( DiagnosticCode.GlobalCollision,
					$"Render attribute \"{global.AttributeName}\" is claimed by both '{owner}' and '{global.Name}'.",
					"An attribute name is the runtime binding key; two parameters sharing one means whichever the engine binds last wins." );
				continue;
			}

			byAttribute[global.AttributeName] = global.Name;
		}

		if ( samplers > PrismConstants.MaxSamplers )
		{
			diagnostics.Warn( DiagnosticCode.BudgetExceeded,
				$"The shader declares {samplers} samplers; the practical limit is {PrismConstants.MaxSamplers}.", null,
				"Reuse the shared samplers from common_samplers.fxc where the filtering matches." );
		}
	}

	static void ValidateVaryings( IrModule module, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var semantics = new HashSet<string>( StringComparer.OrdinalIgnoreCase );
		var names = new HashSet<string>( StringComparer.Ordinal );

		foreach ( var varying in module.Varyings )
		{
			if ( varying is null ) continue;

			if ( !IsValidIdentifier( varying.Name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"'{varying.Name}' is not a legal interpolator name.", null );
				continue;
			}

			if ( !names.Add( varying.Name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"Two interpolators are both named '{varying.Name}'.", null );
				continue;
			}

			// The semantic the writer will actually emit, not the allocator's own register index —
			// IrVarying.Slot counts from zero inside Prism's range and slot 0 is spelled TEXCOORD13.
			var semantic = SboxShaderTemplates.VaryingSemantic( varying );
			var index = SboxShaderTemplates.TexCoordIndex( semantic );

			if ( index >= 0 && index < PrismConstants.FirstFreeTexcoord )
			{
				error( DiagnosticCode.InvalidBlock,
					$"Interpolator '{varying.Name}' claims {semantic}.",
					$"common/pixelinput.hlsl already consumes TEXCOORD0-7 and TEXCOORD11, and Prism reserves 8-10 for its own geometry channels, so allocation starts at TEXCOORD{PrismConstants.FirstFreeTexcoord}." );
				continue;
			}

			if ( !string.IsNullOrEmpty( semantic ) && !semantics.Add( semantic ) )
			{
				error( DiagnosticCode.InvalidBlock,
					$"Interpolator '{varying.Name}' reuses the semantic '{semantic}'.", null );
			}
		}

		if ( module.Varyings.Count > PrismConstants.MaxVaryingSlots )
		{
			error( DiagnosticCode.BudgetExceeded,
				$"The graph needs {module.Varyings.Count} interpolators but only {PrismConstants.MaxVaryingSlots} slots are free.",
				"Compute more of the value in the pixel stage, or pack several values into one float4 varying." );
		}
	}

	static void ValidateStructure( IrModule module, ErrorReporter error, DiagnosticSink diagnostics )
	{
		var meta = module.Meta;

		foreach ( var stage in meta.Stages.Stages() )
		{
			if ( !BackendCapabilities.Sbox.Supports( stage ) )
			{
				error( DiagnosticCode.BackendUnsupported,
					$"The {stage.DisplayName().ToLowerInvariant()} stage cannot be written to a .shader file.",
					"The engine's block parser accepts only VS, PS, GS, CS, PS_RENDER_STATE and RTX." );
				continue;
			}

			if ( module.EntryPoint( stage ) is null && stage != ShaderStage.Vertex )
			{
				diagnostics.Warn( DiagnosticCode.NoOutput,
					$"The module declares the {stage.DisplayName().ToLowerInvariant()} stage but carries no {stage.EntryPoint()} function." );
			}
		}

		if ( meta.Domain == ShaderDomain.Compute && module.EntryPoint( ShaderStage.Compute ) is null )
		{
			error( DiagnosticCode.NoOutput,
				"A compute graph has no compute entry point.",
				$"The engine looks for a function named {PrismConstants.EntryPointCompute} inside a CS block." );
		}

		var structs = new HashSet<string>( StringComparer.Ordinal );

		foreach ( var structure in module.Structs )
		{
			if ( structure is null ) continue;

			if ( !IsValidIdentifier( structure.Name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"'{structure.Name}' is not a legal struct name.", null );
				continue;
			}

			if ( !structs.Add( structure.Name ) )
			{
				error( DiagnosticCode.InvalidBlock, $"Two structs are both named '{structure.Name}'.", null );
			}
		}

		foreach ( var helper in module.Helpers )
		{
			if ( helper is null ) continue;

			if ( !IsValidIdentifier( helper.Name ) )
			{
				error( DiagnosticCode.HelperCollision, $"'{helper.Name}' is not a legal helper function name.", null );
			}
		}
	}
}