Editor/Drivers/SyntheticUiDriver.cs

Editor-side utility that simulates user interaction with Sandbox UI panels. It locates a target Panel by coordinate, numeric path, or simple selector, synthesizes mouse and keyboard events (click, mousedown, mouseup, hover, type), and returns a ToolOutcome describing the result.

File AccessNetworking
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Sandbox;
using Sandbox.UI;

namespace Editor.Mcp
{
	internal static class SyntheticUiDriver
	{
		public static async Task<ToolOutcome<object>> InteractAsync( string selector, string action = "click", string? value = null, string button = "left" )
		{
			if ( !Game.IsPlaying )
			{
				return ToolOutcome<object>.Failure( null, ToolStatus.Unavailable, "ui.playmode_inactive", "Play mode must be active to interact with runtime UI. Call play_start first." );
			}

			var rootPanels = EditorUtility.GetRootPanels();
			if ( rootPanels == null || rootPanels.Count == 0 )
			{
				return ToolOutcome<object>.Failure( null, ToolStatus.Unavailable, "ui.no_root_panels", "No active root panels found in current scene." );
			}

			Panel? targetPanel = null;
			Vector2? coordinateTarget = null;

			// Check if selector is coordinate format: "@x,y" or "x,y"
			var trimmed = (selector ?? "").Trim().TrimStart( '@' );
			var coordParts = trimmed.Split( new[] { ',', 'x', ' ' }, StringSplitOptions.RemoveEmptyEntries );
			if ( coordParts.Length == 2 &&
				float.TryParse( coordParts[0], System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var px ) &&
				float.TryParse( coordParts[1], System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var py ) )
			{
				coordinateTarget = new Vector2( px, py );
				Sandbox.Input.EnableVirtualCursor = true;

				// Hit test panels from top to bottom
				foreach ( var rootObj in rootPanels )
				{
					if ( rootObj is not Panel root || !root.IsValid() ) continue;
					targetPanel = FindPanelAtPosition( root, coordinateTarget.Value );
					if ( targetPanel != null ) break;
				}

				if ( targetPanel == null && rootPanels.Count > 0 )
				{
					targetPanel = rootPanels.FirstOrDefault() as Panel;
				}
			}
			else
			{
				foreach ( var rootObj in rootPanels )
				{
					if ( rootObj is not Panel root || !root.IsValid() ) continue;
					if ( !string.IsNullOrEmpty( selector ) && (selector.StartsWith( "/" ) || selector.All( c => char.IsDigit( c ) || c == '/' )) )
					{
						targetPanel = ResolveNumericPath( root, selector );
					}
					else
					{
						targetPanel = FindBySelector( root, selector ?? "" );
					}
					if ( targetPanel != null ) break;
				}
			}

			if ( targetPanel == null )
			{
				var candidates = rootPanels
					.OfType<Panel>()
					.Where( r => r.IsValid() )
					.SelectMany( r => r.Descendants )
					.Take( 30 )
					.Select( p => p.ElementName )
					.Distinct()
					.ToArray();
				return ToolOutcome<object>.Failure( new { Selector = selector, AvailableSampleElements = candidates }, ToolStatus.Failed, "ui.panel_not_found", $"No panel matched selector '{selector}'." );
			}

			var center = coordinateTarget ?? targetPanel.Box.Rect.Center;
			var buttonStr = button.ToLowerInvariant() switch
			{
				"right" => "right",
				"middle" => "middle",
				_ => "left"
			};

			switch ( action.ToLowerInvariant() )
			{
				case "click":
					targetPanel.CreateEvent( new MousePanelEvent( "onclick", targetPanel, buttonStr ) { LocalPosition = center } );
					break;
				case "mousedown":
					targetPanel.CreateEvent( new MousePanelEvent( "onmousedown", targetPanel, buttonStr ) { LocalPosition = center } );
					break;
				case "mouseup":
					targetPanel.CreateEvent( new MousePanelEvent( "onmouseup", targetPanel, buttonStr ) { LocalPosition = center } );
					break;
				case "hover":
					targetPanel.CreateEvent( new MousePanelEvent( "onmouseover", targetPanel, buttonStr ) { LocalPosition = center } );
					break;
				case "type":
					if ( targetPanel is TextEntry entry )
					{
						entry.Text = value ?? "";
						entry.CreateEvent( "onchange", value, null );
						entry.CreateEvent( "onsubmit", value, null );
					}
					else
					{
						targetPanel.CreateEvent( "onchange", value, null );
					}
					break;
				default:
					return ToolOutcome<object>.Failure( null, ToolStatus.Failed, "ui.invalid_action", $"Unsupported action '{action}'. Use click, mousedown, mouseup, hover, or type." );
			}

			await Task.Delay( 50 );
			await MainThread.Wait();

			return ToolOutcome<object>.Ready( new
			{
				Selector = selector,
				Action = action,
				Panel = targetPanel.ElementName,
				Id = targetPanel.Id,
				Rect = new { targetPanel.Box.Rect.Left, targetPanel.Box.Rect.Top, targetPanel.Box.Rect.Width, targetPanel.Box.Rect.Height },
				Verified = true
			} );
		}

		private static Panel? ResolveNumericPath( Panel root, string path )
		{
			var parts = path.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
			var current = root;
			foreach ( var part in parts )
			{
				if ( !int.TryParse( part, out var idx ) ) return null;
				if ( idx < 0 || idx >= current.ChildrenCount ) return null;
				current = current.GetChild( idx );
				if ( current == null ) return null;
			}
			return current;
		}

		private static Panel? FindBySelector( Panel root, string selector )
		{
			var parts = selector.Split( ' ', StringSplitOptions.RemoveEmptyEntries );
			if ( parts.Length == 0 ) return null;

			IEnumerable<Panel> current = new[] { root };
			foreach ( var part in parts )
			{
				current = current.SelectMany( p => p.Descendants.Prepend( p ) )
					.Where( p => MatchesSimpleSelector( p, part ) )
					.Distinct();
			}

			return current.FirstOrDefault();
		}

		private static readonly Regex SelectorRegex = new( @"^([a-zA-Z0-9_-]+)?((?:[.#][a-zA-Z0-9_-]+)*)$", RegexOptions.Compiled );
		private static readonly Regex ClassOrIdRegex = new( @"([.#])([a-zA-Z0-9_-]+)", RegexOptions.Compiled );

		private static bool MatchesSimpleSelector( Panel panel, string selectorPart )
		{
			if ( string.IsNullOrWhiteSpace( selectorPart ) ) return false;

			var match = SelectorRegex.Match( selectorPart );
			if ( !match.Success ) return false;

			var tag = match.Groups[1].Value;
			if ( !string.IsNullOrEmpty( tag ) && !string.Equals( panel.ElementName, tag, StringComparison.OrdinalIgnoreCase ) )
			{
				return false;
			}

			var modifiers = match.Groups[2].Value;
			if ( string.IsNullOrEmpty( modifiers ) )
			{
				return true;
			}

			var modMatches = ClassOrIdRegex.Matches( modifiers );
			foreach ( Match m in modMatches )
			{
				var prefix = m.Groups[1].Value;
				var val = m.Groups[2].Value;
				if ( prefix == "#" )
				{
					if ( !string.Equals( panel.Id, val, StringComparison.OrdinalIgnoreCase ) )
						return false;
				}
				else if ( prefix == "." )
				{
					if ( !panel.HasClass( val ) )
						return false;
				}
			}

			return true;
		}

		private static Panel? FindPanelAtPosition( Panel root, Vector2 pos )
		{
			if ( !root.IsValid() || !root.IsVisible ) return null;

			// Traverse children in reverse (topmost rendered elements first)
			for ( int i = root.ChildrenCount - 1; i >= 0; i-- )
			{
				var child = root.GetChild( i );
				if ( child != null && child.IsValid() && child.IsVisible )
				{
					var hit = FindPanelAtPosition( child, pos );
					if ( hit != null ) return hit;
				}
			}

			if ( root.Box.Rect.IsInside( pos ) )
			{
				return root;
			}

			return null;
		}
	}
}