Editor/Prism/Toolchain/SlangcValidator.cs

Editor tool that runs the external slangc compiler to validate Slang shader source and to extract reflection JSON. It builds command lines, executes slangc as a child process (optionally feeding source via stdin), parses machine-readable diagnostics or falls back to a plain parser, and returns Diagnostics or reflection data.

Process ExecutionFile Access
using Editor.Prism.Core;
using System.Diagnostics;
using System.IO;
using System.Text;

namespace Editor.Prism.Toolchain;

/// <summary>The raw outcome of one <c>slangc</c> invocation.</summary>
public sealed record SlangcRun( int ExitCode, string StandardOutput, string StandardError )
{
	/// <summary>False when the process could not be started at all.</summary>
	public bool Started { get; init; } = true;

	/// <summary>True when the process outlived its timeout and was killed.</summary>
	public bool TimedOut { get; init; }

	/// <summary>Why the invocation could not run, when it could not.</summary>
	public string Failure { get; init; }

	/// <summary>True when slangc ran to completion and reported success.</summary>
	public bool Ok => Started && !TimedOut && ExitCode == 0;

	/// <summary>Everything the process wrote, in the order a human would read it.</summary>
	public string AllOutput =>
		string.Concat( StandardError ?? string.Empty, StandardError is null ? string.Empty : "\n",
			StandardOutput ?? string.Empty ).Trim();

	/// <inheritdoc/>
	public override string ToString() => Started
		? $"exit {ExitCode}{( TimedOut ? " (timed out)" : string.Empty )}"
		: $"did not start: {Failure}";
}

/// <summary>
/// Drives <c>slangc.exe</c> out of process to type-check Slang.
/// <para>
/// Out of process on purpose. <c>slang.dll</c> is P/Invokable and would give in-process diagnostics
/// with no install, but a native fault in a compiler linked into the editor takes unsaved work with
/// it, whereas a child process just returns a non-zero exit code — and the engine already owns a Slang
/// session of its own inside <c>vfx_vulkan.dll</c> that we would rather not sit beside.
/// </para>
/// <para>
/// Three details of the 2026.14.1 CLI shape everything here. Diagnostics are only parseable in the
/// machine-readable TSV form, because the default output is rustc-style rich text with caret art.
/// Emitting HLSL needs an explicit <c>-entry</c> and <c>-stage</c>, and several entry points in one
/// invocation fails — so one process per entry point. And an unsaved buffer can be fed through stdin
/// with <c>-lang slang ... -- -</c>, which is how the text editor validates without touching disk.
/// </para>
/// <para>
/// Slang validation is Tier 2: it never gates rendering. Slang's own errors therefore arrive as
/// warnings unless <see cref="ReportErrorsAsErrors"/> is turned on, and nothing in this class throws.
/// </para>
/// </summary>
public sealed class SlangcValidator : ISlangValidator
{
	/// <summary>Severity words slangc uses for a primary diagnostic row.</summary>
	static readonly string[] s_primary = { "error", "warning", "fatal error", "internal error" };

	/// <summary>Pseudo-severities slangc uses for rows that belong to the diagnostic above them.</summary>
	static readonly string[] s_attached = { "span", "note-span", "note" };

	/// <summary>Use whatever toolchain <see cref="SlangToolchain"/> located.</summary>
	public SlangcValidator() : this( SlangToolchain.ExecutablePath, SlangToolchain.Version )
	{
	}

	/// <summary>Use a specific executable, e.g. one the user pointed at in Preferences.</summary>
	public SlangcValidator( string executablePath, string version = null )
	{
		ExecutablePath = string.IsNullOrWhiteSpace( executablePath ) ? null : executablePath;
		Version = version;
	}

	/// <summary>Absolute path of the <c>slangc</c> this validator drives, or null.</summary>
	public string ExecutablePath { get; }

	/// <inheritdoc/>
	public bool Available => !string.IsNullOrWhiteSpace( ExecutablePath );

	/// <inheritdoc/>
	public string Version { get; }

	/// <summary>How long one invocation may take before it is killed. Generous; slangc is fast.</summary>
	public int TimeoutMs { get; set; } = 20000;

	/// <summary>
	/// When false — the default, and the contract — a Slang error is reported as a warning, because
	/// Tier 2 never gates rendering. A strict mode can turn it on.
	/// </summary>
	public bool ReportErrorsAsErrors { get; set; }

	/// <summary>Extra arguments appended to every invocation. For debugging the driver itself.</summary>
	public IReadOnlyList<string> ExtraArguments { get; set; } = Array.Empty<string>();

	/// <summary>
	/// Type-check a request, one invocation per entry point, and return the diagnostics. Never throws:
	/// a missing toolchain, a crashed process or unparseable output all come back as an empty or
	/// informational list.
	/// </summary>
	public async Task<IReadOnlyList<Diagnostic>> Validate( SlangValidationRequest request, CancellationToken ct )
	{
		if ( request is null || !Available ) return Array.Empty<Diagnostic>();

		var display = DisplayName( request );
		var results = new List<Diagnostic>();
		var seen = new HashSet<string>( StringComparer.Ordinal );

		var entries = request.EntryPoints is { Count: > 0 }
			? request.EntryPoints.ToArray()
			: new[] { default( SlangEntryPoint ) };

		foreach ( var entry in entries )
		{
			if ( ct.IsCancellationRequested ) break;

			var arguments = BuildArguments( request, entry, null );
			var run = await Execute( arguments, request.Source, WorkingDirectory( request ), ct )
				.ConfigureAwait( false );

			foreach ( var diagnostic in Interpret( run, display ) )
			{
				// One module error repeats across every entry point; show it once.
				var key = $"{diagnostic.Code}|{diagnostic.Span}|{diagnostic.Message}";

				if ( !seen.Add( key ) ) continue;

				results.Add( diagnostic );
			}
		}

		return results;
	}

	/// <summary>
	/// Run slangc for its reflection JSON and parse it. Used to cross-check that the module Prism
	/// emitted exposes exactly the parameter set the graph declares.
	/// </summary>
	public async Task<SlangReflection> Reflect( SlangValidationRequest request, CancellationToken ct )
	{
		if ( request is null || !Available ) return SlangReflection.Empty;

		var path = Path.Combine( Path.GetTempPath(), $"prism-slang-{Guid.NewGuid():N}.json" );

		try
		{
			var entry = request.EntryPoints is { Count: > 0 } ? request.EntryPoints[0] : default;
			var arguments = BuildArguments( request with { Reflection = true }, entry, path );

			await Execute( arguments, request.Source, WorkingDirectory( request ), ct ).ConfigureAwait( false );

			return SlangReflectionReader.ReadFile( path );
		}
		catch ( OperationCanceledException )
		{
			return SlangReflection.Empty;
		}
		catch ( Exception e )
		{
			PrismLog.Error( e, "Reading Slang reflection failed" );

			return SlangReflection.Empty;
		}
		finally
		{
			PrismLog.Guard( "Removing the Slang reflection temp file", () =>
			{
				if ( File.Exists( path ) ) File.Delete( path );
			} );
		}
	}

	// ---- command line ----------------------------------------------------

	/// <summary>
	/// Build the argument list for one entry point. Exposed so the exact command line can be shown in
	/// the diagnostics panel and reproduced by hand.
	/// </summary>
	public IReadOnlyList<string> BuildArguments( SlangValidationRequest request, SlangEntryPoint entry,
		string reflectionPath )
	{
		var arguments = new List<string>();

		if ( request is null ) return arguments;

		var fromStdin = !string.IsNullOrEmpty( request.Source );

		// A real file goes first; stdin is named at the very end after `--`.
		if ( !fromStdin && !string.IsNullOrWhiteSpace( request.FilePath ) ) arguments.Add( request.FilePath );

		arguments.Add( "-target" );
		arguments.Add( string.IsNullOrWhiteSpace( request.Target ) ? "hlsl" : request.Target );

		if ( !string.IsNullOrWhiteSpace( request.LanguageVersion ) )
		{
			arguments.Add( "-std" );
			arguments.Add( request.LanguageVersion );
		}

		// -stage applies to the nearest preceding -entry, so they must stay adjacent and in this order.
		if ( !string.IsNullOrWhiteSpace( entry.Name ) )
		{
			arguments.Add( "-entry" );
			arguments.Add( entry.Name );

			if ( entry.Stage != ShaderStage.None )
			{
				arguments.Add( "-stage" );
				arguments.Add( SlangStageName( entry.Stage ) );
			}
		}

		if ( request.NoCodegen ) arguments.Add( "-no-codegen" );

		arguments.Add( "-enable-machine-readable-diagnostics" );
		arguments.Add( "-diagnostic-color" );
		arguments.Add( "never" );

		foreach ( var include in request.IncludePaths ?? Array.Empty<string>() )
		{
			if ( string.IsNullOrWhiteSpace( include ) ) continue;

			arguments.Add( "-I" );
			arguments.Add( include );
		}

		foreach ( var define in request.Defines ?? Array.Empty<string>() )
		{
			if ( string.IsNullOrWhiteSpace( define ) ) continue;

			arguments.Add( "-D" + define );
		}

		if ( !string.IsNullOrWhiteSpace( reflectionPath ) )
		{
			arguments.Add( "-reflection-json" );
			arguments.Add( reflectionPath );
		}

		foreach ( var extra in ExtraArguments ?? Array.Empty<string>() )
		{
			if ( !string.IsNullOrWhiteSpace( extra ) ) arguments.Add( extra );
		}

		if ( fromStdin )
		{
			// stdin carries no extension, so the language has to be stated; diagnostics then say <stdin>.
			arguments.Add( "-lang" );
			arguments.Add( "slang" );
			arguments.Add( "--" );
			arguments.Add( "-" );
		}

		return arguments;
	}

	/// <summary>Slang's spelling of a stage. Its pixel stage is called <c>fragment</c>.</summary>
	public static string SlangStageName( ShaderStage stage ) => stage switch
	{
		ShaderStage.Vertex => "vertex",
		ShaderStage.Pixel => "fragment",
		ShaderStage.Geometry => "geometry",
		ShaderStage.Compute => "compute",
		_ => "fragment"
	};

	static string DisplayName( SlangValidationRequest request )
	{
		if ( !string.IsNullOrEmpty( request.Source ) )
		{
			return string.IsNullOrWhiteSpace( request.DisplayName ) ? "<stdin>" : request.DisplayName;
		}

		return string.IsNullOrWhiteSpace( request.FilePath ) ? "<stdin>" : request.FilePath;
	}

	string WorkingDirectory( SlangValidationRequest request )
	{
		if ( !string.IsNullOrWhiteSpace( request.FilePath ) )
		{
			var directory = PrismLog.Guard( "Resolving the slangc working directory",
				() => Path.GetDirectoryName( Path.GetFullPath( request.FilePath ) ), null );

			if ( !string.IsNullOrWhiteSpace( directory ) ) return directory;
		}

		return Path.GetDirectoryName( ExecutablePath ) ?? string.Empty;
	}

	// ---- process ---------------------------------------------------------

	/// <summary>
	/// Run slangc once. Standard output and error are drained concurrently — reading them in sequence
	/// deadlocks as soon as either pipe fills — and cancellation kills the process tree.
	/// </summary>
	public async Task<SlangcRun> Execute( IReadOnlyList<string> arguments, string standardInput,
		string workingDirectory, CancellationToken ct )
	{
		if ( !Available )
		{
			return new SlangcRun( -1, null, null ) { Started = false, Failure = "No slangc executable." };
		}

		Process process = null;

		try
		{
			var info = new ProcessStartInfo( ExecutablePath )
			{
				RedirectStandardOutput = true,
				RedirectStandardError = true,
				RedirectStandardInput = standardInput is not null,
				UseShellExecute = false,
				CreateNoWindow = true,
				StandardOutputEncoding = Encoding.UTF8,
				StandardErrorEncoding = Encoding.UTF8,
				WorkingDirectory = workingDirectory ?? string.Empty
			};

			foreach ( var argument in arguments ?? Array.Empty<string>() )
			{
				info.ArgumentList.Add( argument );
			}

			process = Process.Start( info );

			if ( process is null )
			{
				return new SlangcRun( -1, null, null ) { Started = false, Failure = "slangc would not start." };
			}

			using var timeout = CancellationTokenSource.CreateLinkedTokenSource( ct );

			timeout.CancelAfter( Math.Max( 1000, TimeoutMs ) );

			if ( standardInput is not null )
			{
				await process.StandardInput.WriteAsync( standardInput.AsMemory(), timeout.Token )
					.ConfigureAwait( false );

				process.StandardInput.Close();
			}

			var stdout = process.StandardOutput.ReadToEndAsync( timeout.Token );
			var stderr = process.StandardError.ReadToEndAsync( timeout.Token );

			await process.WaitForExitAsync( timeout.Token ).ConfigureAwait( false );

			return new SlangcRun( process.ExitCode,
				await stdout.ConfigureAwait( false ),
				await stderr.ConfigureAwait( false ) );
		}
		catch ( OperationCanceledException )
		{
			Kill( process );

			// A cancellation the caller asked for is not a failure worth reporting to the user.
			return ct.IsCancellationRequested
				? new SlangcRun( -1, null, null ) { Started = false, Failure = "Cancelled." }
				: new SlangcRun( -1, null, null ) { TimedOut = true, Failure = "slangc timed out." };
		}
		catch ( Exception e )
		{
			Kill( process );
			PrismLog.Error( e, "Running slangc failed" );

			return new SlangcRun( -1, null, null ) { Started = false, Failure = e.Message };
		}
		finally
		{
			process?.Dispose();
		}
	}

	static void Kill( Process process )
	{
		if ( process is null ) return;

		PrismLog.Guard( "Killing slangc", () =>
		{
			if ( !process.HasExited ) process.Kill( true );
		} );
	}

	// ---- diagnostics -----------------------------------------------------

	/// <summary>Turn one invocation's output into diagnostics, falling back gracefully at every step.</summary>
	IReadOnlyList<Diagnostic> Interpret( SlangcRun run, string displayName )
	{
		if ( run is null ) return Array.Empty<Diagnostic>();

		if ( !run.Started )
		{
			// Cancellation is silent; anything else is worth one informational line.
			if ( string.Equals( run.Failure, "Cancelled.", StringComparison.Ordinal ) )
			{
				return Array.Empty<Diagnostic>();
			}

			return new[]
			{
				Diagnostic.Info( DiagnosticCode.SlangUnavailable,
					"Slang validation could not run", null, run.Failure )
			};
		}

		if ( run.TimedOut )
		{
			return new[]
			{
				Diagnostic.Info( DiagnosticCode.SlangUnavailable,
					"Slang validation timed out and was cancelled", null,
					"slangc did not finish within its time budget. The module is still emitted; only the " +
					"independent check was skipped." )
			};
		}

		var text = run.AllOutput;

		if ( string.IsNullOrWhiteSpace( text ) ) return Array.Empty<Diagnostic>();

		var parsed = ParseMachineReadable( text, displayName, ReportErrorsAsErrors );

		if ( parsed.Count > 0 ) return parsed;

		// No machine-readable rows. Either slangc is older than we expect or it failed before it could
		// produce any — either way the text still has to reach the user.
		if ( run.Ok ) return Array.Empty<Diagnostic>();

		var fallback = CompilerOutputParser.Parse( text, displayName );
		var results = new List<Diagnostic>( fallback.Count );

		foreach ( var diagnostic in fallback )
		{
			results.Add( diagnostic with
			{
				Code = DiagnosticCode.SlangDiagnostic,
				Severity = Downgrade( diagnostic.Severity )
			} );
		}

		return results;
	}

	DiagnosticSeverity Downgrade( DiagnosticSeverity severity ) =>
		severity == DiagnosticSeverity.Error && !ReportErrorsAsErrors ? DiagnosticSeverity.Warning : severity;

	/// <summary>
	/// Parse <c>-enable-machine-readable-diagnostics</c> output.
	/// <para>
	/// The format is
	/// <c>E&lt;code&gt;\t&lt;severity&gt;\t&lt;file&gt;\t&lt;beginLine&gt;\t&lt;beginCol&gt;\t&lt;endLine&gt;\t&lt;endCol&gt;\t&lt;message&gt;</c>,
	/// and one complaint emits several rows: the primary, then <c>span</c> rows carrying the detailed
	/// explanation, then <c>note</c> and <c>note-span</c> rows. They are folded into one diagnostic with
	/// the explanation as its detail, because eight list entries for one type error is not a diagnostic
	/// panel, it is a wall.
	/// </para>
	/// </summary>
	public static IReadOnlyList<Diagnostic> ParseMachineReadable( string text, string displayName,
		bool errorsAreErrors = false )
	{
		var results = new List<Diagnostic>();

		if ( string.IsNullOrWhiteSpace( text ) ) return results;

		StringBuilder detail = null;
		var index = -1;

		void Flush()
		{
			if ( detail is null || index < 0 ) return;

			var body = detail.ToString().TrimEnd();

			if ( body.Length > 0 )
			{
				var existing = results[index];

				results[index] = existing with
				{
					Detail = string.IsNullOrWhiteSpace( existing.Detail ) ? body : existing.Detail + "\n" + body
				};
			}

			detail = null;
		}

		foreach ( var raw in text.Replace( "\r\n", "\n" ).Split( '\n' ) )
		{
			if ( string.IsNullOrWhiteSpace( raw ) ) continue;

			var fields = raw.Split( '\t', 8 );

			if ( fields.Length < 8 ) continue;

			var code = fields[0].Trim();
			var severity = fields[1].Trim().ToLowerInvariant();
			var file = string.IsNullOrWhiteSpace( fields[2] ) ? displayName : fields[2].Trim();
			var message = fields[7].TrimEnd();

			if ( s_attached.Contains( severity ) )
			{
				if ( index < 0 ) continue;

				detail ??= new StringBuilder();
				detail.AppendLine( severity == "span" ? message : $"{severity}: {message}" );

				continue;
			}

			if ( !s_primary.Contains( severity ) ) continue;

			Flush();

			var beginLine = Number( fields[3], 1 );
			var beginColumn = Number( fields[4], 1 );
			var endLine = Number( fields[5], beginLine );
			var endColumn = Number( fields[6], beginColumn );

			var mapped = severity switch
			{
				"warning" => DiagnosticSeverity.Warning,
				"error" or "fatal error" or "internal error" =>
					errorsAreErrors ? DiagnosticSeverity.Error : DiagnosticSeverity.Warning,
				_ => DiagnosticSeverity.Info
			};

			results.Add( new Diagnostic( mapped, DiagnosticCode.SlangDiagnostic,
				message.Length == 0 ? "slangc reported a problem" : message,
				$"slangc {severity} {code}.",
				new SourceSpan( file, beginLine, beginColumn, Math.Max( endLine, beginLine ), endColumn ),
				null ) );

			index = results.Count - 1;
		}

		Flush();

		return results;
	}

	static int Number( string text, int fallback ) =>
		int.TryParse( text?.Trim(), out var value ) && value > 0 ? value : fallback;

	/// <inheritdoc/>
	public override string ToString() =>
		Available ? $"slangc {Version ?? "?"}" : "slangc unavailable";
}