Code/Brush/TileBrush2x2Edge.cs

This class implements a tile brush that selects one of sixteen predefined sprites based on the connectivity of surrounding tiles. It calculates the appropriate tile index by checking neighbors for cardinal and diagonal connections.

using Sandbox;
using System;
using System.Collections.Generic;
using Saandy.Tilemapper;

namespace Saandy.Tilemapper;

/// <summary>
/// This is the old "3x3 simple" brush renamed to what it functionally is:
/// a compact 2x2-style edge/blob autotile brush using 16 ordered sprites.
/// </summary>
public class TileBrush2x2Edge : TileBrush
{
	[Flags]
	public enum TileMask
	{
		Left = 1 << 0,
		Up = 1 << 1,
		Right = 1 << 2,
		Down = 1 << 3,

		UpLeft = 1 << 4,
		UpRight = 1 << 5,
		DownRight = 1 << 6,
		DownLeft = 1 << 7,
	}

	public override string Name { get; protected set; } = "2x2 Edge Brush";

	private const TileMask CardinalMask =
		TileMask.Left |
		TileMask.Up |
		TileMask.Right |
		TileMask.Down;

	// Tile order:
	// 0  Full
	// 1  Edge Left
	// 2  Edge Up
	// 3  Edge Right
	// 4  Edge Down
	// 5  Horizontal / Left + Right
	// 6  Vertical / Up + Down
	// 7  Outer Top-Right
	// 8  Outer Top-Left
	// 9  Outer Bottom-Left
	// 10 Outer Bottom-Right
	// 11 Inner Top-Left
	// 12 Inner Top-Right
	// 13 Inner Bottom-Right
	// 14 Inner Bottom-Left
	// 15 Isolated / empty-neighbor island
	private const ushort TileFull = 0;

	private const ushort TileLeft = 1;
	private const ushort TileUp = 2;
	private const ushort TileRight = 3;
	private const ushort TileDown = 4;

	private const ushort TileLeftRight = 5;
	private const ushort TileUpDown = 6;

	private const ushort TileOuterTopRight = 7;
	private const ushort TileOuterTopLeft = 8;
	private const ushort TileOuterBottomLeft = 9;
	private const ushort TileOuterBottomRight = 10;

	private const ushort TileInnerTopLeft = 11;
	private const ushort TileInnerTopRight = 12;
	private const ushort TileInnerBottomRight = 13;
	private const ushort TileInnerBottomLeft = 14;

	private const ushort TileIsolated = 15;

	public override ushort GetSpriteIndexToSet( TileMap tilemap, TilesetResource tileset, int x, int y )
	{
		TileMask mask = GetMask( tilemap, x, y );
		TileMask cardinal = mask & CardinalMask;

		bool left = Has( cardinal, TileMask.Left );
		bool up = Has( cardinal, TileMask.Up );
		bool right = Has( cardinal, TileMask.Right );
		bool down = Has( cardinal, TileMask.Down );

		bool upLeft = Has( mask, TileMask.UpLeft );
		bool upRight = Has( mask, TileMask.UpRight );
		bool downRight = Has( mask, TileMask.DownRight );
		bool downLeft = Has( mask, TileMask.DownLeft );

		// Connected on all four cardinal sides.
		// Only in this case do diagonals decide whether we need inner-corner sprites.
		if ( left && up && right && down )
		{
			bool missingUpLeft = !upLeft;
			bool missingUpRight = !upRight;
			bool missingDownRight = !downRight;
			bool missingDownLeft = !downLeft;

			// Diagonal pair cuts. Reuse the straight pieces in the compact 16-tile set.
			if ( missingUpRight && missingDownLeft && !missingUpLeft && !missingDownRight )
				return TileUpDown;

			if ( missingUpLeft && missingDownRight && !missingUpRight && !missingDownLeft )
				return TileLeftRight;

			if ( missingUpLeft && !missingUpRight && !missingDownRight && !missingDownLeft )
				return TileInnerTopLeft;

			if ( missingUpRight && !missingUpLeft && !missingDownRight && !missingDownLeft )
				return TileInnerTopRight;

			if ( missingDownRight && !missingUpLeft && !missingUpRight && !missingDownLeft )
				return TileInnerBottomRight;

			if ( missingDownLeft && !missingUpLeft && !missingUpRight && !missingDownRight )
				return TileInnerBottomLeft;

			return TileFull;
		}

		int cardinalValue = (int)cardinal;

		return cardinalValue switch
		{
			0 => TileIsolated,

			// Side / cap pieces.
			1 => TileLeft,
			2 => TileUp,
			4 => TileRight,
			8 => TileDown,

			// Straight pieces.
			1 | 4 => TileLeftRight,
			2 | 8 => TileUpDown,

			// Outer corners.
			1 | 8 => TileOuterTopRight,     // Left + Down
			4 | 8 => TileOuterTopLeft,      // Right + Down
			2 | 4 => TileOuterBottomLeft,   // Up + Right
			1 | 2 => TileOuterBottomRight,  // Left + Up

			// Three-sided pieces. The compact 16-tile set reuses side pieces for these.
			1 | 2 | 4 => TileDown,  // missing Down
			2 | 4 | 8 => TileLeft,  // missing Left
			4 | 8 | 1 => TileUp,    // missing Up
			8 | 1 | 2 => TileRight, // missing Right

			_ => TileIsolated
		};
	}

	private TileMask GetMask( TileMap tilemap, int x, int y )
	{
		ushort centerTilesetId = tilemap.GetTilesetIdAt( x, y );

		if ( centerTilesetId == 0 )
			return 0;

		TileMask mask = 0;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y ) )
			mask |= TileMask.Left;

		if ( IsSameTileset( tilemap, centerTilesetId, x, y - 1 ) )
			mask |= TileMask.Up;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y ) )
			mask |= TileMask.Right;

		if ( IsSameTileset( tilemap, centerTilesetId, x, y + 1 ) )
			mask |= TileMask.Down;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y - 1 ) )
			mask |= TileMask.UpLeft;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y - 1 ) )
			mask |= TileMask.UpRight;

		if ( IsSameTileset( tilemap, centerTilesetId, x + 1, y + 1 ) )
			mask |= TileMask.DownRight;

		if ( IsSameTileset( tilemap, centerTilesetId, x - 1, y + 1 ) )
			mask |= TileMask.DownLeft;

		return mask;
	}

	private static bool IsSameTileset( TileMap tilemap, ushort centerTilesetId, int x, int y )
	{
		return tilemap.IsSameTilesetId( centerTilesetId, tilemap.GetTilesetIdAt( x, y ) );
	}

	private static bool Has( TileMask mask, TileMask flag )
	{
		return (mask & flag) == flag;
	}

	public override (bool IsVisible, bool UseCollider) GetDefaultTileFlags( ushort spriteIndex )
	{
		// The isolated state is a logical placeholder, not an assignable sprite.
		if ( spriteIndex == TileIsolated )
			return (false, false);

		return (true, true);
	}

	protected override IEnumerable<Vector2Int> GetAffectedTiles( Vector2Int cell )
	{
		yield return new Vector2Int( cell.x - 1, cell.y );
		yield return new Vector2Int( cell.x, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y );
		yield return new Vector2Int( cell.x, cell.y + 1 );

		yield return new Vector2Int( cell.x - 1, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y - 1 );
		yield return new Vector2Int( cell.x + 1, cell.y + 1 );
		yield return new Vector2Int( cell.x - 1, cell.y + 1 );
	}
}