Editor/Brush/TileBrush2x2Edge.cs

A tile brush implementation for a tilemapper editor, it computes a 2x2-style edge/blob autotile sprite index from neighboring tiles and provides visibility/collision defaults and which nearby cells are affected when painting.

using Sandbox;
using System;
using System.Collections.Generic;
using Saandy.Tilemapper;

namespace Saandy.Editor.Tilemapper;

/// <summary>
/// This is the old "3x3 simple" brush renamed to what it functionally is:
/// a compact 2x2-style edge/blob autotile brush using 16 ordered sprites.
/// </summary>
public class TileBrush2x2Edge : TileBrush
{
	[Flags]
	public enum TileMask
	{
		Left = 1 << 0,
		Up = 1 << 1,
		Right = 1 << 2,
		Down = 1 << 3,

		UpLeft = 1 << 4,
		UpRight = 1 << 5,
		DownRight = 1 << 6,
		DownLeft = 1 << 7,
	}

	public override string Name { get; protected set; } = "2x2 Edge Brush";

	private const TileMask CardinalMask =
		TileMask.Left |
		TileMask.Up |
		TileMask.Right |
		TileMask.Down;

	// Tile order:
	// 0  Full
	// 1  Edge Left
	// 2  Edge Up
	// 3  Edge Right
	// 4  Edge Down
	// 5  Horizontal / Left + Right
	// 6  Vertical / Up + Down
	// 7  Outer Top-Right
	// 8  Outer Top-Left
	// 9  Outer Bottom-Left
	// 10 Outer Bottom-Right
	// 11 Inner Top-Left
	// 12 Inner Top-Right
	// 13 Inner Bottom-Right
	// 14 Inner Bottom-Left
	// 15 Isolated / empty-neighbor island
	private const ushort TileFull = 0;

	private const ushort TileLeft = 1;
	private const ushort TileUp = 2;
	private const ushort TileRight = 3;
	private const ushort TileDown = 4;

	private const ushort TileLeftRight = 5;
	private const ushort TileUpDown = 6;

	private const ushort TileOuterTopRight = 7;
	private const ushort TileOuterTopLeft = 8;
	private const ushort TileOuterBottomLeft = 9;
	private const ushort TileOuterBottomRight = 10;

	private const ushort TileInnerTopLeft = 11;
	private const ushort TileInnerTopRight = 12;
	private const ushort TileInnerBottomRight = 13;
	private const ushort TileInnerBottomLeft = 14;

	private const ushort TileIsolated = 15;

	public override ushort GetSpriteIndexToSet( TileMap tilemap, TilesetResource tileset, int x, int y )
	{
		TileMask mask = GetMask( tilemap, x, y );
		TileMask cardinal = mask & CardinalMask;

		bool left = Has( cardinal, TileMask.Left );
		bool up = Has( cardinal, TileMask.Up );
		bool right = Has( cardinal, TileMask.Right );
		bool down = Has( cardinal, TileMask.Down );

		bool upLeft = Has( mask, TileMask.UpLeft );
		bool upRight = Has( mask, TileMask.UpRight );
		bool downRight = Has( mask, TileMask.DownRight );
		bool downLeft = Has( mask, TileMask.DownLeft );

		// Connected on all four cardinal sides.
		// Only in this case do diagonals decide whether we need inner-corner sprites.
		if ( left && up && right && down )
		{
			bool missingUpLeft = !upLeft;
			bool missingUpRight = !upRight;
			bool missingDownRight = !downRight;
			bool missingDownLeft = !downLeft;

			// Diagonal pair cuts. Reuse the straight pieces in the compact 16-tile set.
			if ( missingUpRight && missingDownLeft && !missingUpLeft && !missingDownRight )
				return TileUpDown;

			if ( missingUpLeft && missingDownRight && !missingUpRight && !missingDownLeft )
				return TileLeftRight;

			if ( missingUpLeft && !missingUpRight && !missingDownRight && !missingDownLeft )
				return TileInnerTopLeft;

			if ( missingUpRight && !missingUpLeft && !missingDownRight && !missingDownLeft )
				return TileInnerTopRight;

			if ( missingDownRight && !missingUpLeft && !missingUpRight && !missingDownLeft )
				return TileInnerBottomRight;

			if ( missingDownLeft && !missingUpLeft && !missingUpRight && !missingDownRight )
				return TileInnerBottomLeft;

			return TileFull;
		}

		int cardinalValue = (int)cardinal;

		return cardinalValue switch
		{
			0 => TileIsolated,

			// Side / cap pieces.
			1 => TileLeft,
			2 => TileUp,
			4 => TileRight,
			8 => TileDown,

			// Straight pieces.
			1 | 4 => TileLeftRight,
			2 | 8 => TileUpDown,

			// Outer corners.
			1 | 8 => TileOuterTopRight,     // Left + Down
			4 | 8 => TileOuterTopLeft,      // Right + Down
			2 | 4 => TileOuterBottomLeft,   // Up + Right
			1 | 2 => TileOuterBottomRight,  // Left + Up

			// Three-sided pieces. The compact 16-tile set reuses side pieces for these.
			1 | 2 | 4 => TileDown,  // missing Down
			2 | 4 | 8 => TileLeft,  // missing Left
			4 | 8 | 1 => TileUp,    // missing Up
			8 | 1 | 2 => TileRight, // missing Right

			_ => TileIsolated
		};
	}

	private TileMask GetMask( TileMap tilemap, int x, int y )
	{
		ushort centerTilesetId = tilemap.GetTilesetIdAt( x, y );

		if ( centerTilesetId == 0 )
			return 0;

		TileMask mask = 0;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y ) )
			mask |= TileMask.Left;

		if ( IsSameTileset( tilemap, centerTilesetId, x, y - 1 ) )
			mask |= TileMask.Up;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y ) )
			mask |= TileMask.Right;

		if ( IsSameTileset( tilemap, centerTilesetId, x, y + 1 ) )
			mask |= TileMask.Down;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y - 1 ) )
			mask |= TileMask.UpLeft;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y - 1 ) )
			mask |= TileMask.UpRight;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y + 1 ) )
			mask |= TileMask.DownRight;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y + 1 ) )
			mask |= TileMask.DownLeft;

		return mask;
	}

	private static bool IsSameTileset( TileMap tilemap, ushort centerTilesetId, int x, int y )
	{
		return tilemap.IsSameTilesetId( centerTilesetId, tilemap.GetTilesetIdAt( x, y ) );
	}

	private static bool Has( TileMask mask, TileMask flag )
	{
		return (mask & flag) == flag;
	}

	public override (bool IsVisible, bool UseCollider) GetDefaultTileFlags( ushort spriteIndex )
	{
		// Slot 15 is the logical isolated placeholder for this compact 2x2 edge set.
		// It must stay in the tilemap so later neighbours can connect to it,
		// but it should not draw and should not generate collision.
		if ( spriteIndex == TileIsolated )
			return (false, false);

		return (true, true);
	}

	protected override IEnumerable<Vector2Int> GetAffectedTiles( Vector2Int cell )
	{
		yield return new Vector2Int( cell.x - 1, cell.y );
		yield return new Vector2Int( cell.x, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y );
		yield return new Vector2Int( cell.x, cell.y + 1 );

		yield return new Vector2Int( cell.x - 1, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y + 1 );
		yield return new Vector2Int( cell.x - 1, cell.y + 1 );
	}
}