Editor/ModalTransformOperation.cs

Editor modal transform base class. Manages lifecycle, input locking, selection snapshot, constraint state, numeric input, and confirms/cancels for translate/rotate/scale modal operations used in the scene editor.

File Access
#nullable enable
using Editor;
using Sandbox;
using System;
using System.Collections.Generic;
using System.Linq;

namespace BlenderActions;

/// <summary>Defines world-axis and world-plane transform constraints.</summary>
[Flags]
public enum AxisConstraint
{
    None = 0,
    X = 1,
    Y = 2,
    Z = 4,
    XY = X | Y,
    XZ = X | Z,
    YZ = Y | Z
}

/// <summary>Identifies the supported modal transform operation types.</summary>
public enum TransformOperationKind
{
    Translate,
    Rotate,
    Scale
}

/// <summary>Provides the shared lifecycle, input handling, and cleanup for modal transforms.</summary>
public abstract class ModalTransformOperation
{
    /// <summary>Defines the frame delay that prevents the completing click from reaching default viewport input.</summary>
    private const int PostClickGuardFrames = 2;
    /// <summary>Defines pointer-motion scaling while the precision modifier is held.</summary>
    protected const float PrecisionMultiplier = 0.1f;

    /// <summary>Owns temporary viewport input suppression for this operation.</summary>
    private ViewportInputLock? _inputLock;
    /// <summary>References the scene view bound to the active operation.</summary>
    private SceneViewWidget? _sceneView;
    /// <summary>References the scene viewport bound to the active operation.</summary>
    private SceneViewportWidget? _viewport;
    /// <summary>References the editor session bound to the active operation.</summary>
    private SceneEditorSession? _session;
    /// <summary>References the editor camera bound to the active operation.</summary>
    private CameraComponent? _camera;
    /// <summary>Handles selection snapshot.</summary>
    private GameObject[] _selectionSnapshot = Array.Empty<GameObject>();
    /// <summary>Tracks whether the operation is waiting to finish.</summary>
    private bool _finishRequested;
    /// <summary>Tracks whether numeric or modal confirmation has been requested.</summary>
    private bool _confirmRequested;
    /// <summary>Tracks whether the modal operation has already terminated.</summary>
    private bool _finished;
    /// <summary>Stores remaining frames used to guard the confirming or cancelling click.</summary>
    private int _finishDelayFrames;

    /// <summary>Gets the bound scene view or throws when unavailable.</summary>
    protected SceneViewWidget SceneView =>
        _sceneView ?? throw new InvalidOperationException("No active Scene View.");
    /// <summary>Gets the bound scene viewport or throws when unavailable.</summary>
    protected SceneViewportWidget Viewport =>
        _viewport ?? throw new InvalidOperationException("No active Scene Viewport.");
    /// <summary>Gets the bound scene editor session or throws when unavailable.</summary>
    protected SceneEditorSession Session =>
        _session ?? throw new InvalidOperationException("No active Scene Editor session.");
    /// <summary>Gets the bound editor camera or throws when unavailable.</summary>
    protected CameraComponent Camera =>
        _camera ?? throw new InvalidOperationException("No active Scene camera.");

    /// <summary>Handles selected objects.</summary>
    protected GameObject[] SelectedObjects { get; private set; } = Array.Empty<GameObject>();
    /// <summary>Handles new.</summary>
    protected internal NumericInputSession NumericInput { get; } = new();
    /// <summary>Gets the active world-axis or world-plane constraint.</summary>
    protected AxisConstraint Constraint { get; private set; }

    /// <summary>Gets the editor session currently bound to this operation.</summary>
    internal SceneEditorSession? BoundSession => _session;
    /// <summary>Gets the operation kind.</summary>
    public abstract TransformOperationKind Kind { get; }

    /// <summary>Binds the operation to the active scene context and starts its modal lifecycle.</summary>
    internal bool Begin(SceneEditorSession expectedSession)
    {
        var sceneView = SceneViewWidget.Current;
        var viewport = sceneView?.LastSelectedViewportWidget;
        var session = SceneEditorSession.Active;

        if(sceneView == null ||
            viewport == null ||
            session == null ||
            !ReferenceEquals(session, expectedSession))
        {
            return false;
        }

        var activeTool = sceneView.Tools.CurrentTool;
        var activeSubTool = sceneView.Tools.CurrentSubTool;
        var camera = activeSubTool?.Camera ?? activeTool?.Camera;

        if(camera == null)
            return false;

        _sceneView = sceneView;
        _viewport = viewport;
        _session = session;
        _camera = camera;

        _selectionSnapshot = session.GetSelection()
            .OfType<GameObject>()
            .Where(gameObject => gameObject.IsValid())
            .ToArray();

        var selectedSet = _selectionSnapshot.ToHashSet();
        SelectedObjects = _selectionSnapshot
            .Where(gameObject => !HasSelectedAncestor(gameObject, selectedSet))
            .ToArray();

        if(SelectedObjects.Length == 0)
        {
            ClearContext();
            return false;
        }

        Constraint = AxisConstraint.None;

        try
        {
            OnBegin();
            _inputLock = new ViewportInputLock(
                SceneView,
                Viewport,
                activeTool,
                activeSubTool);
            SceneView.MouseClick += RequestConfirm;
            SceneView.MouseRightClick += RequestCancel;
            NumericInput.Begin();
            return true;
        }
        catch
        {
            Cleanup();
            throw;
        }
    }

    /// <summary>Advances input, completion, and operation-specific update logic.</summary>
    internal void Tick()
    {
        if(_finished)
            return;

        if(!HasValidContext())
        {
            Abort();
            return;
        }

        if(NumericInput.ConsumeConfirmRequest())
            RequestConfirm();

        if(_finishRequested)
        {
            if(_finishDelayFrames > 0)
            {
                _finishDelayFrames--;
                return;
            }

            Finish(_confirmRequested);
            return;
        }

        try
        {
            OnUpdate();
        }
        catch
        {
            Abort();
            throw;
        }
    }

    /// <summary>Applies or toggles an axis constraint on the active operation.</summary>
    internal void SetConstraint(AxisConstraint constraint)
    {
        if(_finished || _finishRequested)
            return;

        Constraint = Constraint == constraint
            ? AxisConstraint.None
            : constraint;

        OnConstraintChanged();
    }

    /// <summary>Queues confirmation after the viewport click guard interval.</summary>
    internal void RequestConfirm()
    {
        if(_finished || _finishRequested)
            return;

        _confirmRequested = true;
        _finishRequested = true;
        _finishDelayFrames = PostClickGuardFrames;
    }

    /// <summary>Restores initial state and queues cancellation after the click guard interval.</summary>
    internal void RequestCancel()
    {
        if(_finished || _finishRequested)
            return;

        RestoreInitialState();
        _confirmRequested = false;
        _finishRequested = true;
        _finishDelayFrames = PostClickGuardFrames;
    }

    /// <summary>Immediately restores initial state and terminates the operation.</summary>
    internal void Abort()
    {
        if(_finished)
            return;

        try
        {
            RestoreInitialState();
        }
        finally
        {
            Finish(false);
        }
    }

    /// <summary>Captures operation-specific initial state.</summary>
    protected abstract void OnBegin();
    /// <summary>Updates the operation from current editor input.</summary>
    protected abstract void OnUpdate();
    /// <summary>Restores every transformed object to its captured initial state.</summary>
    protected abstract void RestoreInitialState();
    /// <summary>Registers undo and redo callbacks for the completed operation.</summary>
    protected abstract void RegisterUndo();
    /// <summary>Resets operation-specific state after the active constraint changes.</summary>
    protected virtual void OnConstraintChanged() { }
    /// <summary>Releases operation-specific state during cleanup.</summary>
    protected virtual void OnCleanup() { }

    /// <summary>Restores the scene selection captured when the operation began.</summary>
    protected void RestoreSelection()
    {
        var session = _session;

        if(session == null)
            return;

        session.Selection.Clear();

        foreach(var gameObject in _selectionSnapshot)
        {
            if(gameObject.IsValid())
                session.Selection.Add(gameObject);
        }
    }

    /// <summary>Converts camera-render pixels to viewport input pixels.</summary>
    protected Vector2 CameraPixelsToInputPixels(Vector2 cameraPixels)
    {
        var renderSize = Camera.CustomSize;
        var inputSize = Viewport.Size * Viewport.DpiScale;

        if(!renderSize.HasValue || renderSize.Value.x <= 0f || renderSize.Value.y <= 0f)
            return cameraPixels;

        return new Vector2(
            cameraPixels.x * inputSize.x / renderSize.Value.x,
            cameraPixels.y * inputSize.y / renderSize.Value.y);
    }

    /// <summary>Converts viewport input pixels to camera-render pixels.</summary>
    protected Vector2 InputPixelsToCameraPixels(Vector2 inputPixels)
    {
        var renderSize = Camera.CustomSize;
        var inputSize = Viewport.Size * Viewport.DpiScale;

        if(!renderSize.HasValue || inputSize.x <= 0f || inputSize.y <= 0f)
            return inputPixels;

        return new Vector2(
            inputPixels.x * renderSize.Value.x / inputSize.x,
            inputPixels.y * renderSize.Value.y / inputSize.y);
    }

    /// <summary>Returns whether a selected ancestor already represents this object.</summary>
    private static bool HasSelectedAncestor(
        GameObject gameObject,
        HashSet<GameObject> selected)
    {
        var parent = gameObject.Parent;

        while(parent != null)
        {
            if(selected.Contains(parent))
                return true;

            parent = parent.Parent;
        }

        return false;
    }

    /// <summary>Returns whether the bound scene, viewport, session, and camera remain valid.</summary>
    private bool HasValidContext()
    {
        return _sceneView != null &&
            _sceneView.IsValid &&
            _viewport != null &&
            _viewport.IsValid &&
            _session != null &&
            SceneEditorSession.Active == _session &&
            _camera != null &&
            _camera.GameObject != null &&
            _camera.GameObject.IsValid();
    }

    /// <summary>Commits or cancels the operation and always releases modal resources.</summary>
    private void Finish(bool confirmed)
    {
        if(_finished)
            return;

        _finished = true;

        try
        {
            if(confirmed)
            {
                try
                {
                    RegisterUndo();
                    Session.HasUnsavedChanges = true;
                }
                catch
                {
                    RestoreInitialState();
                    throw;
                }
            }

            RestoreSelection();
        }
        finally
        {
            ModalOperationArbiter.Release(this);
            Cleanup();
        }
    }

    /// <summary>Unsubscribes input handlers, restores viewport state, and clears context.</summary>
    private void Cleanup()
    {
        Exception? cleanupError = null;

        try
        {
            if(_sceneView != null)
            {
                _sceneView.MouseClick -= RequestConfirm;
                _sceneView.MouseRightClick -= RequestCancel;
            }
        }
        catch(Exception exception)
        {
            cleanupError ??= exception;
        }

        NumericInput.End();

        try
        {
            _inputLock?.Dispose();
        }
        catch(Exception exception)
        {
            cleanupError ??= exception;
        }

        _inputLock = null;

        try
        {
            OnCleanup();
        }
        catch(Exception exception)
        {
            cleanupError ??= exception;
        }

        ClearContext();

        if(cleanupError != null)
            throw cleanupError;
    }

    /// <summary>Clears references to the active editor context and selection.</summary>
    private void ClearContext()
    {
        _sceneView = null;
        _viewport = null;
        _session = null;
        _camera = null;
        SelectedObjects = Array.Empty<GameObject>();
        _selectionSnapshot = Array.Empty<GameObject>();
    }
}