Editor/WindowDock.cs

Editor dock widget that lists top-level windows from other processes and allows reparenting a selected window into an editor panel using Win32 SetParent. It saves and restores window styles/position, keeps the hosted window sized to the panel, and returns it on exit or destruction.

Native InteropProcess ExecutionReflection
using Editor;
using Sandbox;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;

namespace WindowDock;

/// <summary>
/// Editor dock that hosts another application's window. Pick a window from the list and
/// Dock reparents it into the panel (Win32 SetParent); Release hands it back to the desktop
/// with its original frame. Docked windows are released automatically when the editor exits.
/// </summary>
[Dock( "Editor", "Window Dock", "picture_in_picture" )]
public class ExternalWindowDock : Widget
{
	readonly ComboBox windowList;
	readonly Button dockButton;
	readonly Button releaseButton;
	readonly Label status;
	readonly HostWidget host;

	List<IntPtr> listedWindows = new();

	// The docked window and what we need to put it back the way we found it.
	IntPtr docked;
	long savedStyle;
	long savedExStyle;
	Win32.RECT savedRect;
	IntPtr hostHandle;

	bool mouseWasDown;
	RealTimeSince sinceClickedInside = 100;

	public ExternalWindowDock( Widget parent ) : base( parent )
	{
		Layout = Layout.Column();

		var bar = Layout.AddRow();
		bar.Margin = 4;
		bar.Spacing = 4;

		windowList = bar.Add( new ComboBox( this ), 1 );
		windowList.ToolTip = "Open windows (title — process)";

		var refresh = bar.Add( new IconButton( "refresh", RefreshList, this ) );
		refresh.ToolTip = "Refresh the window list";

		dockButton = bar.Add( new Button( "Dock", "input", this ) );
		dockButton.Clicked = DockSelected;

		releaseButton = bar.Add( new Button( "Release", "output", this ) );
		releaseButton.Clicked = Release;

		var add = bar.Add( new IconButton( "add", OpenAnotherPanel, this ) );
		add.ToolTip = "Open another Window Dock panel";

		// Keystrokes meant for the docked app must never land on these (a stray space would press Dock).
		foreach ( var control in new Widget[] { windowList, refresh, dockButton, releaseButton, add } )
			control.FocusMode = FocusMode.None;

		host = Layout.Add( new HostWidget( this ), 1 );
		host.Resized = FitToHost;

		status = host.Layout.Add( new Label( "Pick a window and press Dock.", host ) );
		status.Alignment = TextFlag.Center;

		RefreshList();
		UpdateButtons();
	}

	void RefreshList()
	{
		listedWindows = Win32.ListDockableWindows( docked );
		windowList.Clear();

		foreach ( var hwnd in listedWindows )
			windowList.AddItem( Win32.Describe( hwnd ) );

		if ( listedWindows.Count == 0 )
			windowList.AddItem( "No dockable windows found", enabled: false );
	}

	/// <summary>
	/// Extra panels are registered with the dock manager at runtime, so they get a View menu entry
	/// for this session but aren't recreated after an editor restart.
	/// </summary>
	void OpenAnotherPanel()
	{
		var manager = EditorWindow.DockManager;

		var number = 2;
		while ( manager.FindDockWidget( $"Window Dock {number}" ) is not null )
			number++;

		var panel = manager.AddDock( $"Window Dock {number}", "picture_in_picture", new ExternalWindowDock( null ), DockArea.Center, manager.FindDockWidget( this ) );
		panel.SetAsCurrentTab();
	}

	void DockSelected()
	{
		// One window per panel; Release first (or use +) so a click can't silently swap it out.
		if ( docked != IntPtr.Zero )
			return;

		var index = windowList.CurrentIndex;
		if ( index < 0 || index >= listedWindows.Count )
			return;

		var target = listedWindows[index];
		if ( !Win32.IsWindow( target ) )
		{
			SetStatus( "That window has closed." );
			RefreshList();
			return;
		}

		hostHandle = NativeHandle( host );
		if ( hostHandle == IntPtr.Zero )
		{
			SetStatus( "Couldn't get the panel's native window handle (s&box internals changed?)." );
			return;
		}

		// Restore a maximized/minimized window first so the saved rect is its normal size.
		if ( Win32.IsZoomed( target ) || Win32.IsIconic( target ) )
			Win32.ShowWindow( target, Win32.SW_RESTORE );

		savedStyle = Win32.GetWindowLong( target, Win32.GWL_STYLE );
		savedExStyle = Win32.GetWindowLong( target, Win32.GWL_EXSTYLE );
		Win32.GetWindowRect( target, out savedRect );

		// Make sure Qt doesn't paint over the hosted window.
		Win32.SetWindowLong( hostHandle, Win32.GWL_STYLE, Win32.GetWindowLong( hostHandle, Win32.GWL_STYLE ) | Win32.WS_CLIPCHILDREN );

		// Win32 wants WS_CHILD set (and WS_POPUP cleared) before SetParent.
		var style = (savedStyle & ~Win32.FrameStyles) | Win32.WS_CHILD | Win32.WS_VISIBLE;
		var exStyle = savedExStyle & ~Win32.FrameExStyles;
		Win32.SetWindowLong( target, Win32.GWL_STYLE, style );
		Win32.SetWindowLong( target, Win32.GWL_EXSTYLE, exStyle );

		if ( Win32.SetParent( target, hostHandle ) == IntPtr.Zero && Marshal.GetLastWin32Error() != 0 )
		{
			var error = Marshal.GetLastWin32Error();
			Win32.SetWindowLong( target, Win32.GWL_STYLE, savedStyle );
			Win32.SetWindowLong( target, Win32.GWL_EXSTYLE, savedExStyle );
			Win32.SetWindowPos( target, IntPtr.Zero, 0, 0, 0, 0, Win32.SWP_NOMOVE | Win32.SWP_NOSIZE | Win32.SWP_NOZORDER | Win32.SWP_FRAMECHANGED );
			SetStatus( $"Windows refused to dock that window (error {error})." );
			return;
		}

		docked = target;
		status.Visible = false;
		FitToHost();
		Win32.ShowWindow( docked, Win32.SW_SHOW );
		Win32.SetFocus( docked );

		Log.Info( $"Window Dock: docked {Win32.Describe( docked )}" );
		RefreshList();
		UpdateButtons();
	}

	void Release() => Release( true );

	void Release( bool updateUi )
	{
		if ( docked == IntPtr.Zero )
			return;

		var window = docked;
		docked = IntPtr.Zero;

		if ( Win32.IsWindow( window ) )
		{
			Win32.SetParent( window, IntPtr.Zero );
			Win32.SetWindowLong( window, Win32.GWL_STYLE, savedStyle );
			Win32.SetWindowLong( window, Win32.GWL_EXSTYLE, savedExStyle );
			Win32.SetWindowPos( window, IntPtr.Zero, savedRect.Left, savedRect.Top,
				savedRect.Right - savedRect.Left, savedRect.Bottom - savedRect.Top,
				Win32.SWP_NOZORDER | Win32.SWP_FRAMECHANGED | Win32.SWP_SHOWWINDOW );
			Win32.SetForegroundWindow( window );
		}

		if ( !updateUi )
			return;

		SetStatus( "Pick a window and press Dock." );
		RefreshList();
		UpdateButtons();
	}

	void FitToHost()
	{
		if ( docked == IntPtr.Zero )
			return;

		if ( hostHandle == IntPtr.Zero || !Win32.GetClientRect( hostHandle, out var client ) )
			return;

		Win32.MoveWindow( docked, 0, 0, client.Right, client.Bottom, true );
	}

	/// <summary>
	/// Apps that draw their own title bar (Electron, Chromium) drag and resize themselves, and their
	/// minimize/restore buttons still work. Keep the window filling the panel whatever it does.
	/// </summary>
	void HoldInPlace()
	{
		if ( Win32.IsIconic( docked ) )
			Win32.ShowWindow( docked, Win32.SW_RESTORE );

		// Ends the drag/resize loop the app started from its title bar.
		if ( Win32.IsInMoveSize( docked ) )
			Win32.PostMessage( docked, Win32.WM_CANCELMODE, IntPtr.Zero, IntPtr.Zero );

		Win32.GetWindowRect( docked, out var window );
		Win32.GetWindowRect( hostHandle, out var panel );
		if ( window.Left != panel.Left || window.Top != panel.Top || window.Right != panel.Right || window.Bottom != panel.Bottom )
			FitToHost();
	}

	/// <summary>
	/// Clicking the docked window activates the editor, and Qt then pulls keyboard focus back to
	/// itself. For a moment after a click inside, hand focus back to the docked window.
	/// Clicking back into the editor doesn't return focus by itself: Qt still thinks the widget it
	/// last focused (e.g. the game viewport) has it, so it never asks Windows for it. Do that for Qt.
	/// </summary>
	void FocusOnClick()
	{
		var down = Win32.AnyMouseButtonDown();
		if ( down && !mouseWasDown )
		{
			if ( Win32.IsUnderCursor( docked ) )
			{
				sinceClickedInside = 0;
			}
			else
			{
				sinceClickedInside = 100;

				if ( Win32.HasFocus( docked ) )
					Win32.FocusEditorWindowUnderCursor();
			}
		}

		mouseWasDown = down;

		if ( sinceClickedInside < 0.3f && !Win32.HasFocus( docked ) )
			Win32.SetFocus( docked );
	}

	void SetStatus( string text )
	{
		status.Text = text;
		status.Visible = docked == IntPtr.Zero;
	}

	void UpdateButtons()
	{
		dockButton.Enabled = docked == IntPtr.Zero;
		releaseButton.Enabled = docked != IntPtr.Zero;
	}

	[EditorEvent.Frame]
	void KeepDocked()
	{
		if ( docked == IntPtr.Zero )
			return;

		// The hosted app closed its window.
		if ( !Win32.IsWindow( docked ) )
		{
			docked = IntPtr.Zero;
			SetStatus( "The docked window was closed." );
			RefreshList();
			UpdateButtons();
			return;
		}

		// A hotload adds new fields at their default.
		if ( hostHandle == IntPtr.Zero )
			hostHandle = NativeHandle( host );

		HoldInPlace();
		FocusOnClick();
	}

	// Destroying the editor's windows would take a docked window down with them,
	// so hand it back while ours still exist.
	[Event( "app.exit" )]
	void ReleaseOnExit() => Release( false );

	// Best effort: by now Qt has usually destroyed our native window (and the child with it).
	public override void OnDestroyed()
	{
		Release( false );
		base.OnDestroyed();
	}

	/// <summary>
	/// Editor.Widget keeps its QWidget internal; read it and ask Qt for the HWND (this also makes the widget native).
	/// </summary>
	static IntPtr NativeHandle( Widget widget )
	{
		var field = typeof( Widget ).GetField( "_widget", BindingFlags.Instance | BindingFlags.NonPublic );
		var native = field?.GetValue( widget );
		var winId = native?.GetType().GetMethod( "winId", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, Type.EmptyTypes );

		return winId?.Invoke( native, null ) is IntPtr handle ? handle : IntPtr.Zero;
	}

	class HostWidget : Widget
	{
		public Action Resized;

		public HostWidget( Widget parent ) : base( parent )
		{
			Layout = Layout.Column();
			SetSizeMode( SizeMode.Flexible, SizeMode.Flexible );
		}

		protected override void OnResize()
		{
			base.OnResize();
			Resized?.Invoke();
		}
	}

	static class Win32
	{
		public const int GWL_STYLE = -16;
		public const int GWL_EXSTYLE = -20;

		public const long WS_CHILD = 0x40000000;
		public const long WS_VISIBLE = 0x10000000;
		public const long WS_CLIPCHILDREN = 0x02000000;
		const long WS_POPUP = 0x80000000;
		const long WS_CAPTION = 0x00C00000;
		const long WS_THICKFRAME = 0x00040000;
		const long WS_SYSMENU = 0x00080000;
		const long WS_MINIMIZEBOX = 0x00020000;
		const long WS_MAXIMIZEBOX = 0x00010000;
		public const long FrameStyles = WS_POPUP | WS_CAPTION | WS_THICKFRAME | WS_SYSMENU | WS_MINIMIZEBOX | WS_MAXIMIZEBOX;

		const long WS_EX_DLGMODALFRAME = 0x00000001;
		const long WS_EX_TOOLWINDOW = 0x00000080;
		const long WS_EX_WINDOWEDGE = 0x00000100;
		const long WS_EX_CLIENTEDGE = 0x00000200;
		const long WS_EX_STATICEDGE = 0x00020000;
		const long WS_EX_APPWINDOW = 0x00040000;
		public const long FrameExStyles = WS_EX_DLGMODALFRAME | WS_EX_WINDOWEDGE | WS_EX_CLIENTEDGE | WS_EX_STATICEDGE | WS_EX_APPWINDOW;

		public const int SW_SHOW = 5;
		public const int SW_RESTORE = 9;

		public const uint SWP_NOSIZE = 0x0001;
		public const uint SWP_NOMOVE = 0x0002;
		public const uint SWP_NOZORDER = 0x0004;
		public const uint SWP_FRAMECHANGED = 0x0020;
		public const uint SWP_SHOWWINDOW = 0x0040;

		public const uint WM_CANCELMODE = 0x001F;

		const uint GW_OWNER = 4;
		const int DWMWA_CLOAKED = 14;
		const int GUI_INMOVESIZE = 0x0002;

		[StructLayout( LayoutKind.Sequential )]
		public struct RECT { public int Left, Top, Right, Bottom; }

		[StructLayout( LayoutKind.Sequential )]
		struct POINT { public int X, Y; }

		[StructLayout( LayoutKind.Sequential )]
		struct GUITHREADINFO
		{
			public int cbSize;
			public int flags;
			public IntPtr hwndActive, hwndFocus, hwndCapture, hwndMenuOwner, hwndMoveSize, hwndCaret;
			public RECT rcCaret;
		}

		delegate bool EnumWindowsProc( IntPtr hwnd, IntPtr lParam );

		[DllImport( "user32.dll" )] static extern bool EnumWindows( EnumWindowsProc callback, IntPtr lParam );
		[DllImport( "user32.dll" )] static extern bool IsWindowVisible( IntPtr hwnd );
		[DllImport( "user32.dll" )] static extern IntPtr GetWindow( IntPtr hwnd, uint cmd );
		[DllImport( "user32.dll" )] static extern int GetWindowTextLength( IntPtr hwnd );
		[DllImport( "user32.dll", CharSet = CharSet.Unicode )] static extern int GetWindowText( IntPtr hwnd, StringBuilder text, int max );
		[DllImport( "user32.dll", CharSet = CharSet.Unicode )] static extern int GetClassName( IntPtr hwnd, StringBuilder text, int max );
		[DllImport( "user32.dll" )] static extern uint GetWindowThreadProcessId( IntPtr hwnd, out uint processId );
		[DllImport( "user32.dll", EntryPoint = "GetWindowLongPtrW" )] static extern IntPtr GetWindowLongPtr( IntPtr hwnd, int index );
		[DllImport( "user32.dll", EntryPoint = "SetWindowLongPtrW" )] static extern IntPtr SetWindowLongPtr( IntPtr hwnd, int index, IntPtr value );
		[DllImport( "dwmapi.dll" )] static extern int DwmGetWindowAttribute( IntPtr hwnd, int attribute, out int value, int size );

		[DllImport( "user32.dll" )] public static extern bool IsWindow( IntPtr hwnd );
		[DllImport( "user32.dll" )] public static extern bool IsZoomed( IntPtr hwnd );
		[DllImport( "user32.dll" )] public static extern bool IsIconic( IntPtr hwnd );
		[DllImport( "user32.dll" )] public static extern bool ShowWindow( IntPtr hwnd, int cmd );
		[DllImport( "user32.dll" )] public static extern bool GetWindowRect( IntPtr hwnd, out RECT rect );
		[DllImport( "user32.dll", SetLastError = true )] public static extern IntPtr SetParent( IntPtr child, IntPtr parent );
		[DllImport( "user32.dll" )] public static extern bool MoveWindow( IntPtr hwnd, int x, int y, int width, int height, bool repaint );
		[DllImport( "user32.dll" )] public static extern bool SetWindowPos( IntPtr hwnd, IntPtr after, int x, int y, int width, int height, uint flags );
		[DllImport( "user32.dll" )] public static extern bool SetForegroundWindow( IntPtr hwnd );
		[DllImport( "user32.dll" )] public static extern bool GetClientRect( IntPtr hwnd, out RECT rect );
		[DllImport( "user32.dll" )] public static extern bool PostMessage( IntPtr hwnd, uint msg, IntPtr wParam, IntPtr lParam );
		[DllImport( "user32.dll" )] public static extern IntPtr SetFocus( IntPtr hwnd );
		[DllImport( "user32.dll" )] static extern IntPtr GetFocus();
		[DllImport( "user32.dll" )] static extern bool IsChild( IntPtr parent, IntPtr hwnd );
		[DllImport( "user32.dll" )] static extern bool GetCursorPos( out POINT point );
		[DllImport( "user32.dll" )] static extern IntPtr WindowFromPoint( POINT point );
		[DllImport( "user32.dll" )] static extern short GetAsyncKeyState( int key );
		[DllImport( "user32.dll" )] static extern bool GetGUIThreadInfo( uint threadId, ref GUITHREADINFO info );

		static bool IsSelfOrChild( IntPtr window, IntPtr hwnd ) => hwnd != IntPtr.Zero && (hwnd == window || IsChild( window, hwnd ));

		public static bool HasFocus( IntPtr window ) => IsSelfOrChild( window, GetFocus() );

		public static bool IsUnderCursor( IntPtr window ) => GetCursorPos( out var point ) && IsSelfOrChild( window, WindowFromPoint( point ) );

		/// <summary>
		/// Give keyboard focus to the editor window under the cursor (the game viewport, the main window...).
		/// Clicks on other apps or other docked windows are left alone.
		/// </summary>
		public static void FocusEditorWindowUnderCursor()
		{
			if ( !GetCursorPos( out var point ) )
				return;

			var hwnd = WindowFromPoint( point );
			if ( hwnd == IntPtr.Zero )
				return;

			GetWindowThreadProcessId( hwnd, out var pid );
			if ( pid == (uint)Environment.ProcessId )
				SetFocus( hwnd );
		}

		// Left, right and middle buttons.
		public static bool AnyMouseButtonDown() => ((GetAsyncKeyState( 0x01 ) | GetAsyncKeyState( 0x02 ) | GetAsyncKeyState( 0x04 )) & 0x8000) != 0;

		public static bool IsInMoveSize( IntPtr window )
		{
			var info = new GUITHREADINFO { cbSize = Marshal.SizeOf<GUITHREADINFO>() };
			return GetGUIThreadInfo( GetWindowThreadProcessId( window, out _ ), ref info )
				&& (info.flags & GUI_INMOVESIZE) != 0 && IsSelfOrChild( window, info.hwndMoveSize );
		}

		public static long GetWindowLong( IntPtr hwnd, int index ) => GetWindowLongPtr( hwnd, index ).ToInt64();
		public static void SetWindowLong( IntPtr hwnd, int index, long value ) => SetWindowLongPtr( hwnd, index, new IntPtr( value ) );

		/// <summary>
		/// Visible, titled, unowned top-level windows from other processes - roughly what the taskbar shows.
		/// </summary>
		public static List<IntPtr> ListDockableWindows( IntPtr include )
		{
			var ownProcess = (uint)Environment.ProcessId;
			var windows = new List<IntPtr>();

			EnumWindows( ( hwnd, _ ) =>
			{
				if ( hwnd == include )
					return true;

				if ( !IsWindowVisible( hwnd ) || GetWindowTextLength( hwnd ) == 0 || GetWindow( hwnd, GW_OWNER ) != IntPtr.Zero )
					return true;

				if ( (GetWindowLong( hwnd, GWL_EXSTYLE ) & WS_EX_TOOLWINDOW) != 0 )
					return true;

				GetWindowThreadProcessId( hwnd, out var pid );
				if ( pid == ownProcess )
					return true;

				// Suspended/background UWP frames report visible but are cloaked.
				if ( DwmGetWindowAttribute( hwnd, DWMWA_CLOAKED, out var cloaked, sizeof( int ) ) == 0 && cloaked != 0 )
					return true;

				// UWP/WinUI hosts (Calculator, Settings, Terminal...) can't be reparented properly.
				var className = ClassName( hwnd );
				if ( className is "ApplicationFrameWindow" or "Windows.UI.Core.CoreWindow" or "Progman" )
					return true;

				windows.Add( hwnd );
				return true;
			}, IntPtr.Zero );

			return windows;
		}

		public static string Describe( IntPtr hwnd )
		{
			var title = new StringBuilder( GetWindowTextLength( hwnd ) + 1 );
			GetWindowText( hwnd, title, title.Capacity );

			GetWindowThreadProcessId( hwnd, out var pid );
			string process;
			try { process = Process.GetProcessById( (int)pid ).ProcessName; }
			catch { process = $"pid {pid}"; }

			return $"{title} — {process}";
		}

		static string ClassName( IntPtr hwnd )
		{
			var name = new StringBuilder( 256 );
			GetClassName( hwnd, name, name.Capacity );
			return name.ToString();
		}
	}
}