Editor/Prism/Integration/CodeFileEditor.cs

Editor integration for Prism code editor. Declares which file extensions are treated as shader sources, provides PrismCodeEditor implementing ICodeEditor to open shader files in Prism while delegating non-shader files to a fallback editor, and hooks into asset browser and editor preferences to route .shader/.hlsl/.slang files into Prism.

File AccessNetworking
using Editor.Prism.Core;
using System.IO;

namespace Editor.Prism.Integration;

/// <summary>
/// Which files Prism's code window is willing to own.
/// <para>
/// Shader sources only. C#, Razor and SCSS belong to a real IDE and Prism never claims them, which is
/// what makes it safe to let Prism act as the editor-wide code editor.
/// </para>
/// </summary>
public static class PrismShaderFiles
{
	/// <summary>Extensions, without the leading dot, that Prism opens as shader source.</summary>
	public static readonly IReadOnlyList<string> Extensions = new[]
	{
		PrismConstants.ShaderExtension,   // shader — the engine's VFX block format
		PrismConstants.HlslExtension,     // hlsl
		"hlsli",
		"fxc",
		PrismConstants.SlangExtension,    // slang
		"slangh",
		"vfx"
	};

	/// <summary>True when Prism's code window is the right place for this path.</summary>
	public static bool IsShaderSource( string path )
	{
		if ( string.IsNullOrWhiteSpace( path ) ) return false;

		var extension = Path.GetExtension( path );

		if ( string.IsNullOrEmpty( extension ) ) return false;

		extension = extension.TrimStart( '.' );

		foreach ( var candidate in Extensions )
		{
			if ( extension.Equals( candidate, StringComparison.OrdinalIgnoreCase ) ) return true;
		}

		return false;
	}

	/// <summary>A name filter string suitable for <see cref="FileDialog.SetNameFilter"/>.</summary>
	public static string NameFilter =>
		"Shader Source (" + string.Join( " ", Extensions.Select( x => $"*.{x}" ) ) + ")";
}

/// <summary>
/// Prism as an editor-wide code editor, offered but never imposed.
/// <para>
/// Any type implementing <c>ICodeEditor</c> is listed in <i>Editor Settings ▸ General ▸ Code Editor</i>
/// automatically, so this shows up as a choice the moment the assembly loads. Selecting it routes
/// shader sources into Prism's code window; everything else — C#, Razor, SCSS, solutions, addons — is
/// handed straight to whichever editor was selected before, so picking Prism never costs you your IDE.
/// </para>
/// </summary>
[Title( "Prism" ), Icon( "gradient" )]
public sealed class PrismCodeEditor : ICodeEditor
{
	/// <summary>
	/// Always available: it ships inside the editor assembly, so unlike an external IDE there is
	/// nothing to find on disk. Note that selecting it only takes over <em>shader</em> sources —
	/// everything else is forwarded to <see cref="CodeFileEditor.Fallback"/>, which is why this is not
	/// gated on one existing.
	/// </summary>
	public bool IsInstalled() => true;

	/// <summary>Shader sources open in Prism; everything else goes to the fallback editor.</summary>
	public void OpenFile( string path, int? line = null, int? column = null )
	{
		if ( string.IsNullOrWhiteSpace( path ) ) return;

		if ( PrismShaderFiles.IsShaderSource( path ) )
		{
			PrismLauncher.OpenCode( path, line ?? 0, column ?? 1 );

			return;
		}

		var fallback = CodeFileEditor.Fallback;

		if ( fallback is null )
		{
			// Prism is a shader editor; a .cs file has to go somewhere else. Saying so beats a
			// double-click that appears to do nothing at all.
			PrismLog.Warn( $"Prism cannot open '{Path.GetFileName( path )}' — it edits shader sources " +
				"only, and no other code editor is available to hand it to. Pick one in " +
				"Editor Settings ▸ Code Editor." );

			return;
		}

		fallback.OpenFile( path, line, column );
	}

	/// <summary>Prism has no notion of a solution. Delegated.</summary>
	public void OpenSolution() => CodeFileEditor.Fallback?.OpenSolution();

	/// <summary>Prism has no notion of an addon workspace. Delegated.</summary>
	public void OpenAddon( Project addon ) => CodeFileEditor.Fallback?.OpenAddon( addon );
}

/// <summary>
/// Routes shader text files into Prism's code window.
/// <para>
/// Three separate paths reach a text file in this editor, and none of them can be intercepted the
/// same way:
/// </para>
/// <list type="number">
/// <item><description><c>.shader</c> is a native asset type whose <c>OpenInEditor</c> short-circuits
/// to <c>EditorEvent.Run( "open.shader", path )</c> before <c>IAssetEditor</c> is ever consulted, so
/// the only hook is the event — which is multicast and uncancellable, meaning the tools addon still
/// launches VS Code alongside us if it is installed. Hence the preference.</description></item>
/// <item><description><c>.hlsl</c> and <c>.slang</c> cannot be registered as asset types at all; they
/// arrive as plain files through the asset browser's <c>OnFileSelected</c> delegate, which we chain
/// rather than replace.</description></item>
/// <item><description>Anything routed through <c>CodeEditor.OpenFile</c> reaches
/// <see cref="PrismCodeEditor"/>, but only if the user opted in.</description></item>
/// </list>
/// </summary>
public static class CodeFileEditor
{
	static ICodeEditor s_fallback;
	static Action<string> s_previousFileSelected;
	static AssetBrowser s_routedBrowser;

	/// <summary>
	/// The editor Prism hands non-shader files to. Resolved lazily, cached until hotload, and never
	/// resolves to Prism itself.
	/// </summary>
	public static ICodeEditor Fallback
	{
		get
		{
			s_fallback ??= ResolveFallback();

			return s_fallback;
		}
	}

	/// <summary>Friendly name of the fallback editor, for the preferences page.</summary>
	public static string FallbackTitle =>
		PrismLog.Guard( "Describing the fallback code editor",
			() => Fallback?.Title, null ) ?? "no external editor";

	/// <summary>True when Prism is currently the editor-wide code editor.</summary>
	public static bool IsCurrentCodeEditor =>
		PrismLog.Guard( "Reading the current code editor",
			() => CodeEditor.Current is PrismCodeEditor, false );

	// ---- the .shader event ------------------------------------------------

	/// <summary>
	/// Double-clicking a <c>.shader</c> lands here. Runs early so Prism is up before any external
	/// editor steals focus.
	/// </summary>
	[Event( "open.shader", Priority = -100 )]
	public static void OnOpenShader( string absolutePath )
	{
		if ( !PrismCookies.ClaimShaderFiles ) return;
		if ( string.IsNullOrWhiteSpace( absolutePath ) ) return;

		PrismLauncher.OpenCode( absolutePath );
	}

	// ---- asset browser routing --------------------------------------------

	/// <summary>
	/// Chain ourselves onto the asset browser's plain-file handler, so an unregistered
	/// <c>.hlsl</c>/<c>.slang</c> opens in Prism instead of the operating system's shell handler.
	/// <para>
	/// Idempotent, and safe to call every frame: it re-installs if the browser is recreated or if
	/// something else has overwritten the delegate since.
	/// </para>
	/// </summary>
	public static void EnsureAssetBrowserRouting()
	{
		PrismLog.Guard( "Routing plain files through Prism", () =>
		{
			var local = MainAssetBrowser.Instance?.Local;

			if ( local is null || !local.IsValid ) return;
			if ( ReferenceEquals( s_routedBrowser, local ) && IsOurs( local.OnFileSelected ) ) return;

			var previous = local.OnFileSelected;

			// Never chain to ourselves — after a hotload the delegate sitting there is our own
			// handler from the outgoing assembly, and chaining would grow a new link every reload.
			s_previousFileSelected = IsOurs( previous ) ? null : previous;
			s_routedBrowser = local;
			local.OnFileSelected = OnFileSelected;
		} );
	}

	/// <summary>Hand the plain-file handler back to whoever had it. Called when the preference goes off.</summary>
	public static void RemoveAssetBrowserRouting()
	{
		PrismLog.Guard( "Restoring the asset browser file handler", () =>
		{
			var local = MainAssetBrowser.Instance?.Local;

			if ( local is null || !local.IsValid ) return;
			if ( !IsOurs( local.OnFileSelected ) ) return;

			local.OnFileSelected = s_previousFileSelected ?? ( f => EditorUtility.OpenFile( f ) );
			s_routedBrowser = null;
			s_previousFileSelected = null;
		} );
	}

	static void OnFileSelected( string absolutePath )
	{
		if ( PrismCookies.ClaimShaderFiles && PrismShaderFiles.IsShaderSource( absolutePath ) )
		{
			PrismLauncher.OpenCode( absolutePath );

			return;
		}

		if ( PrismAssetEditor.IsPrismDocument( absolutePath ) )
		{
			PrismAssetEditor.Open( absolutePath );

			return;
		}

		if ( s_previousFileSelected is not null )
		{
			s_previousFileSelected( absolutePath );

			return;
		}

		// Same behaviour MainAssetBrowser installs by default.
		PrismLog.Guard( "Opening a file with the shell handler", () => EditorUtility.OpenFile( absolutePath ) );
	}

	/// <summary>
	/// A delegate is one of ours when it was declared on this type — compared by full name, so it
	/// still matches an instance left behind by the previous assembly.
	/// </summary>
	static bool IsOurs( Action<string> handler )
	{
		var declaring = handler?.Method?.DeclaringType;

		return declaring is not null
			&& string.Equals( declaring.FullName, typeof( CodeFileEditor ).FullName, StringComparison.Ordinal );
	}

	// ---- the editor-wide code editor preference ---------------------------

	static bool s_reconciling;

	/// <summary>
	/// Startup reconciliation.
	/// <para>
	/// If the user picked Prism directly in <i>Editor Settings ▸ Code Editor</i>, that choice wins and
	/// the preference is updated to match — reverting it would be the tool arguing with the person
	/// using it. Otherwise the preference is applied.
	/// </para>
	/// </summary>
	public static void ApplyCodeEditorPreference()
	{
		PrismLog.Guard( "Applying the Prism code editor preference", () =>
		{
			// Read the raw cookie rather than CodeEditor.Current: the getter instantiates the selected
			// editor and probes the filesystem and registry for it, and no editor session should pay
			// that at startup just because Prism happens to be installed.
			var selected = EditorCookie?.GetString( CodeEditorCookie, null );

			if ( string.Equals( selected, nameof( PrismCodeEditor ), StringComparison.Ordinal ) )
			{
				if ( !PrismCookies.RouteCodeFiles ) PrismCookies.RouteCodeFiles = true;

				return;
			}

			if ( !PrismCookies.RouteCodeFiles ) return;

			ReconcileCodeEditorPreference();
		} );
	}

	/// <summary>
	/// The engine's own key for the selected code editor. Hard-coded in <c>CodeEditor.Current</c>, and
	/// stored as the implementing type's short name.
	/// </summary>
	const string CodeEditorCookie = "CodeEditor";

	/// <summary>
	/// Make <c>CodeEditor.Current</c> agree with <see cref="PrismCookies.RouteCodeFiles"/>.
	/// <para>
	/// Turning it on remembers whatever was selected before, so turning it off puts that back rather
	/// than leaving the editor with no code editor at all. Subscribed to
	/// <see cref="PrismCookies.Changed"/>, so flipping the toggle in the preferences page takes effect
	/// immediately.
	/// </para>
	/// </summary>
	public static void ReconcileCodeEditorPreference()
	{
		if ( s_reconciling ) return;

		s_reconciling = true;

		try
		{
			PrismLog.Guard( "Reconciling the Prism code editor preference", () =>
			{
				var current = CodeEditor.Current;
				var wanted = PrismCookies.RouteCodeFiles;

				if ( wanted )
				{
					// Compared by full name, not with `is`. CodeEditor.Current is cached in a private
					// static on Sandbox.Tools, which does not hotload, so after the editor assembly is
					// swapped that field still holds a PrismCodeEditor from the OUTGOING assembly — a
					// different Type identity, so `is` says false. The old code then recorded
					// "PrismCodeEditor" as the user's fallback IDE, permanently, and ResolveFallback
					// excludes PrismCodeEditor by type, so the remembered name could never match again
					// and the user silently got whichever of VisualStudio/VSCode/Rider probed first.
					if ( IsPrism( current ) ) return;

					if ( current is not null )
					{
						PrismCookies.FallbackCodeEditor = current.GetType().Name;
						s_fallback = current;
					}

					CodeEditor.Current = new PrismCodeEditor();

					return;
				}

				if ( !IsPrism( current ) ) return;

				var restored = Fallback;

				if ( restored is not null ) CodeEditor.Current = restored;
			} );
		}
		finally
		{
			s_reconciling = false;
		}
	}

	/// <summary>
	/// Whether an <c>ICodeEditor</c> is Prism's, judged by full type name rather than by type identity.
	/// A hotload leaves an instance of the outgoing assembly's <c>PrismCodeEditor</c> in a static that
	/// does not hotload, and that instance fails <c>is PrismCodeEditor</c> against the new type.
	/// </summary>
	static bool IsPrism( ICodeEditor editor ) =>
		editor is not null &&
		string.Equals( editor.GetType().FullName, typeof( PrismCodeEditor ).FullName, StringComparison.Ordinal );

	/// <summary>Drop the cached fallback so it is resolved again after a hotload or a settings change.</summary>
	public static void FlushFallback()
	{
		s_fallback = null;
	}

	/// <summary>Forget the chained delegate — it points into the outgoing assembly after a hotload.</summary>
	public static void ForgetRouting()
	{
		s_previousFileSelected = null;
		s_routedBrowser = null;
	}

	static ICodeEditor ResolveFallback()
	{
		return PrismLog.Guard( "Resolving the fallback code editor", () =>
		{
			var types = EditorTypeLibrary.GetTypes<ICodeEditor>()
				.Where( x => !x.IsInterface && !x.IsAbstract )
				.Where( x => x.TargetType != typeof( PrismCodeEditor ) )
				.ToList();

			ICodeEditor Instantiate( TypeDescription type )
			{
				var editor = type?.Create<ICodeEditor>();

				return editor is not null && editor.IsInstalled() ? editor : null;
			}

			var remembered = PrismCookies.FallbackCodeEditor;

			if ( !string.IsNullOrWhiteSpace( remembered ) )
			{
				var match = Instantiate( types.FirstOrDefault( x => x.Name == remembered ) );

				if ( match is not null ) return match;
			}

			foreach ( var preferred in new[] { "VisualStudio", "VisualStudioCode", "Rider" } )
			{
				var match = Instantiate( types.FirstOrDefault( x => x.Name == preferred ) );

				if ( match is not null ) return match;
			}

			foreach ( var type in types )
			{
				var match = Instantiate( type );

				if ( match is not null ) return match;
			}

			return null;
		}, null );
	}
}