Editor-side utility that simulates user interaction with Sandbox UI panels. It locates a target Panel by coordinate, numeric path, or simple selector, synthesizes mouse and keyboard events (click, mousedown, mouseup, hover, type), and returns a ToolOutcome describing the result.
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Sandbox;
using Sandbox.UI;
namespace Editor.Mcp
{
internal static class SyntheticUiDriver
{
public static async Task<ToolOutcome<object>> InteractAsync( string selector, string action = "click", string? value = null, string button = "left" )
{
if ( !Game.IsPlaying )
{
return ToolOutcome<object>.Failure( null, ToolStatus.Unavailable, "ui.playmode_inactive", "Play mode must be active to interact with runtime UI. Call play_start first." );
}
var rootPanels = EditorUtility.GetRootPanels();
if ( rootPanels == null || rootPanels.Count == 0 )
{
return ToolOutcome<object>.Failure( null, ToolStatus.Unavailable, "ui.no_root_panels", "No active root panels found in current scene." );
}
Panel? targetPanel = null;
Vector2? coordinateTarget = null;
// Check if selector is coordinate format: "@x,y" or "x,y"
var trimmed = (selector ?? "").Trim().TrimStart( '@' );
var coordParts = trimmed.Split( new[] { ',', 'x', ' ' }, StringSplitOptions.RemoveEmptyEntries );
if ( coordParts.Length == 2 &&
float.TryParse( coordParts[0], System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var px ) &&
float.TryParse( coordParts[1], System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var py ) )
{
coordinateTarget = new Vector2( px, py );
Sandbox.Input.EnableVirtualCursor = true;
// Hit test panels from top to bottom
foreach ( var rootObj in rootPanels )
{
if ( rootObj is not Panel root || !root.IsValid() ) continue;
targetPanel = FindPanelAtPosition( root, coordinateTarget.Value );
if ( targetPanel != null ) break;
}
if ( targetPanel == null && rootPanels.Count > 0 )
{
targetPanel = rootPanels.FirstOrDefault() as Panel;
}
}
else
{
foreach ( var rootObj in rootPanels )
{
if ( rootObj is not Panel root || !root.IsValid() ) continue;
if ( !string.IsNullOrEmpty( selector ) && (selector.StartsWith( "/" ) || selector.All( c => char.IsDigit( c ) || c == '/' )) )
{
targetPanel = ResolveNumericPath( root, selector );
}
else
{
targetPanel = FindBySelector( root, selector ?? "" );
}
if ( targetPanel != null ) break;
}
}
if ( targetPanel == null )
{
var candidates = rootPanels
.OfType<Panel>()
.Where( r => r.IsValid() )
.SelectMany( r => r.Descendants )
.Take( 30 )
.Select( p => p.ElementName )
.Distinct()
.ToArray();
return ToolOutcome<object>.Failure( new { Selector = selector, AvailableSampleElements = candidates }, ToolStatus.Failed, "ui.panel_not_found", $"No panel matched selector '{selector}'." );
}
var center = coordinateTarget ?? targetPanel.Box.Rect.Center;
var buttonStr = button.ToLowerInvariant() switch
{
"right" => "right",
"middle" => "middle",
_ => "left"
};
switch ( action.ToLowerInvariant() )
{
case "click":
targetPanel.CreateEvent( new MousePanelEvent( "onclick", targetPanel, buttonStr ) { LocalPosition = center } );
break;
case "mousedown":
targetPanel.CreateEvent( new MousePanelEvent( "onmousedown", targetPanel, buttonStr ) { LocalPosition = center } );
break;
case "mouseup":
targetPanel.CreateEvent( new MousePanelEvent( "onmouseup", targetPanel, buttonStr ) { LocalPosition = center } );
break;
case "hover":
targetPanel.CreateEvent( new MousePanelEvent( "onmouseover", targetPanel, buttonStr ) { LocalPosition = center } );
break;
case "type":
if ( targetPanel is TextEntry entry )
{
entry.Text = value ?? "";
entry.CreateEvent( "onchange", value, null );
entry.CreateEvent( "onsubmit", value, null );
}
else
{
targetPanel.CreateEvent( "onchange", value, null );
}
break;
default:
return ToolOutcome<object>.Failure( null, ToolStatus.Failed, "ui.invalid_action", $"Unsupported action '{action}'. Use click, mousedown, mouseup, hover, or type." );
}
await Task.Delay( 50 );
await MainThread.Wait();
return ToolOutcome<object>.Ready( new
{
Selector = selector,
Action = action,
Panel = targetPanel.ElementName,
Id = targetPanel.Id,
Rect = new { targetPanel.Box.Rect.Left, targetPanel.Box.Rect.Top, targetPanel.Box.Rect.Width, targetPanel.Box.Rect.Height },
Verified = true
} );
}
private static Panel? ResolveNumericPath( Panel root, string path )
{
var parts = path.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
var current = root;
foreach ( var part in parts )
{
if ( !int.TryParse( part, out var idx ) ) return null;
if ( idx < 0 || idx >= current.ChildrenCount ) return null;
current = current.GetChild( idx );
if ( current == null ) return null;
}
return current;
}
private static Panel? FindBySelector( Panel root, string selector )
{
var parts = selector.Split( ' ', StringSplitOptions.RemoveEmptyEntries );
if ( parts.Length == 0 ) return null;
IEnumerable<Panel> current = new[] { root };
foreach ( var part in parts )
{
current = current.SelectMany( p => p.Descendants.Prepend( p ) )
.Where( p => MatchesSimpleSelector( p, part ) )
.Distinct();
}
return current.FirstOrDefault();
}
private static readonly Regex SelectorRegex = new( @"^([a-zA-Z0-9_-]+)?((?:[.#][a-zA-Z0-9_-]+)*)$", RegexOptions.Compiled );
private static readonly Regex ClassOrIdRegex = new( @"([.#])([a-zA-Z0-9_-]+)", RegexOptions.Compiled );
private static bool MatchesSimpleSelector( Panel panel, string selectorPart )
{
if ( string.IsNullOrWhiteSpace( selectorPart ) ) return false;
var match = SelectorRegex.Match( selectorPart );
if ( !match.Success ) return false;
var tag = match.Groups[1].Value;
if ( !string.IsNullOrEmpty( tag ) && !string.Equals( panel.ElementName, tag, StringComparison.OrdinalIgnoreCase ) )
{
return false;
}
var modifiers = match.Groups[2].Value;
if ( string.IsNullOrEmpty( modifiers ) )
{
return true;
}
var modMatches = ClassOrIdRegex.Matches( modifiers );
foreach ( Match m in modMatches )
{
var prefix = m.Groups[1].Value;
var val = m.Groups[2].Value;
if ( prefix == "#" )
{
if ( !string.Equals( panel.Id, val, StringComparison.OrdinalIgnoreCase ) )
return false;
}
else if ( prefix == "." )
{
if ( !panel.HasClass( val ) )
return false;
}
}
return true;
}
private static Panel? FindPanelAtPosition( Panel root, Vector2 pos )
{
if ( !root.IsValid() || !root.IsVisible ) return null;
// Traverse children in reverse (topmost rendered elements first)
for ( int i = root.ChildrenCount - 1; i >= 0; i-- )
{
var child = root.GetChild( i );
if ( child != null && child.IsValid() && child.IsVisible )
{
var hit = FindPanelAtPosition( child, pos );
if ( hit != null ) return hit;
}
}
if ( root.Box.Rect.IsInside( pos ) )
{
return root;
}
return null;
}
}
}