Code/V2/Runtime/HexagonRuntimeSystem.cs

Scene-scoped runtime system for Hexagon v2. Manages host and client initialization, player session lifecycle, persistence startup/shutdown, and network listener callbacks for connections, disconnections and host changes.

NetworkingFile AccessNative Interop
#nullable enable

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Hexagon.V2.Application;
using Hexagon.V2.Client;
using Hexagon.V2.Composition;
using Hexagon.V2.Domain;
using Hexagon.V2.Infrastructure;
using Hexagon.V2.Kernel;
using Hexagon.V2.Kernel.Schema;
using Hexagon.V2.Networking;
using Hexagon.V2.Persistence;
using Sandbox;

namespace Hexagon.V2.Runtime;

public enum HexRuntimeReadiness
{
	Absent = 0,
	Initializing = 1,
	Ready = 2,
	Failed = 3,
	Disposing = 4,
	Disposed = 5
}

/// <summary>
/// Scene-scoped host/client composition root. Listen servers receive independent
/// scopes, while each host connection owns an ephemeral command and character epoch.
/// </summary>
public sealed class HexagonRuntimeSystem : GameObjectSystem<HexagonRuntimeSystem>, ISceneStartup, Component.INetworkListener
{
	private readonly Scene _runtimeScene;
	private readonly Dictionary<Guid, RuntimePlayerSession<HexPlayerBody>> _sessions = new();
	private readonly SpawnSlotAllocator _spawnSlots = new();
	private readonly CancellationTokenSource _hostLifetime = new();
	private readonly AsyncOperationRegistry _hostOperations = new();
	private Task? _hostInitialization;
	private Task<OperationResult>? _hostShutdown;
	private Task<OperationResult>? _resourceShutdown;
	private IPersistenceProvider? _persistence;
	private CompiledSchema? _hostSchema;
	private HexHostServicesComponent? _hostServices;
	private HexClientRootComponent? _clientRoot;
	private HexClientController? _clientController;
	private bool _hostFailureLogged;
	private bool _disposeRequested;
	private bool _hostLifetimeDisposed;
	private string _persistenceRoot = string.Empty;
	private IHexSpawnSelector? _spawnSelector;

	public HexagonRuntimeSystem( Scene scene ) : base( scene )
	{
		_runtimeScene = scene;
		Listen( Stage.FinishUpdate, 100, PollLifecycle, "Hexagon v2 lifecycle" );
	}

	public HexRuntimeReadiness HostReadiness { get; private set; } = HexRuntimeReadiness.Absent;
	public HexRuntimeReadiness ClientReadiness { get; private set; } = HexRuntimeReadiness.Absent;
	public IHexHostApplication? HostApplication { get; private set; }
	public HexClientStore? ClientStore { get; private set; }
	public IHexClientController? ClientController => _clientController;
	public Task<OperationResult>? HostShutdownCompletion => _hostShutdown ?? _resourceShutdown;
	internal SandboxClientCommandTransport? ClientTransport { get; private set; }

	/// <summary>
	/// The single answer to where a player belongs, used by both connect placement and every
	/// later embodiment. Defaults to <see cref="SceneTagSpawnSelector"/>; a game replaces it
	/// through <c>HexSchemaRuntimeDescriptor.CreateSpawnSelector</c>. Resolved lazily because
	/// connections can arrive before host initialization has run.
	/// </summary>
	internal IHexSpawnSelector SpawnSelector =>
		_spawnSelector ??= new SceneTagSpawnSelector( _runtimeScene, _spawnSlots );

	void ISceneStartup.OnHostInitialize()
	{
		if ( HostReadiness != HexRuntimeReadiness.Absent ) return;
		var bootstrapResult = RequireBootstrap();
		if ( bootstrapResult.Failed )
		{
			FailHost( bootstrapResult.Error!.Message );
			return;
		}
		var bootstrap = bootstrapResult.Value;
		var hostOptions = ResolveHostOptions( bootstrap );
		if ( hostOptions.Failed )
		{
			FailHost( hostOptions.Error!.Message );
			return;
		}

		var collector = new SchemaSourceCollector();
		Scene.RunEvent<IHexSchemaSource>( source => source.CollectSchemas( collector ) );
		var descriptorResult = collector.Require( bootstrap.SchemaId );
		if ( descriptorResult.Failed )
		{
			FailHost( descriptorResult.Error!.Message );
			return;
		}

		if ( descriptorResult.Value.CreateSpawnSelector is { } createSelector )
		{
			try
			{
				_spawnSelector = createSelector( _runtimeScene )
					?? throw new InvalidOperationException( "The spawn selector factory returned null." );
			}
			catch ( Exception exception )
			{
				FailHost( $"Spawn selector construction failed: {exception.Message}" );
				return;
			}
		}

		var compiled = SchemaCompiler.Compile( descriptorResult.Value.Schema );
		if ( compiled.Failed )
		{
			FailHost( compiled.Error!.Message );
			return;
		}

		var types = new PersistedTypeRegistry().RegisterHexagonDomainTypes();
		var bindings = SchemaPersistenceAdapter.Bind(
			compiled.Value, types, descriptorResult.Value.PersistenceCodecs );
		if ( bindings.Failed )
		{
			FailHost( bindings.Error!.Message );
			return;
		}

		IPersistenceStorage storage;
		try
		{
			storage = descriptorResult.Value.CreatePersistenceStorage()
				?? throw new InvalidOperationException( "The schema persistence storage factory returned null." );
		}
		catch ( Exception exception )
		{
			FailHost( $"Persistence storage construction failed: {exception.Message}" );
			return;
		}
		var logicalRoot = $"hexagon/persistence/v3/{compiled.Value.Id}";
		if ( !string.IsNullOrWhiteSpace( hostOptions.Value.PersistenceRootOverride ) )
		{
			storage = new PrefixedPersistenceStorage( storage, hostOptions.Value.PersistenceRootOverride );
			_persistenceRoot = $"{hostOptions.Value.PersistenceRootOverride}/{logicalRoot}";
		}
		else
		{
			_persistenceRoot = logicalRoot;
		}

		var persistenceInvariants = new DomainInvariantValidator(
			bindings.Value.Types,
			compiled.Value,
			new SchemaItemShapeCatalog( compiled.Value ),
			descriptorResult.Value.PersistenceInvariants );
		// One-shot: consume the quarantine arming for THIS host start regardless of outcome, so a
		// start that does not quarantine (intact store, lease-unavailable, a different root) cannot
		// leave it armed to silently quarantine a later store. Re-arm to quarantine again.
		var quarantineCorruptStore = HexagonRuntimeOverrides.QuarantineCorruptStore;
		if ( quarantineCorruptStore ) HexagonRuntimeOverrides.QuarantineCorruptStore = false;
		_persistence = new FileSystemPersistenceProvider(
			storage,
			new FileSystemPersistenceOptions( compiled.Value.Id )
			{
				Log = message => Log.Info( message ),
				// Operator-armed one-shot corruption recovery; the arming was consumed at read above.
				QuarantineCorruptStore = quarantineCorruptStore,
				// Threshold checkpoints must not stall the committing command or the main
				// thread; the registry-tracked task is drained by shutdown before the
				// provider's own final checkpoint runs.
				ScheduleBackgroundCheckpoint = work => TryStartHostOperation(
					"persistence:automatic-checkpoint",
					async () =>
					{
						await GameTask.WorkerThread();
						await work();
					} )
			},
			bindings.Value.Types,
			persistenceInvariants );
		HostReadiness = HexRuntimeReadiness.Initializing;
		_hostSchema = compiled.Value;
		var hostServices = CreateHostServices();
		if ( hostServices.Failed )
		{
			FailHost( hostServices.Error!.Message );
			_hostInitialization = ShutdownResourcesOnceAsync();
			return;
		}
		_hostServices = hostServices.Value;
		_hostInitialization = InitializeHostAsync(
			descriptorResult.Value,
			compiled.Value,
			bindings.Value,
			_persistence,
			_hostServices,
			_runtimeScene,
			_persistenceRoot,
			hostOptions.Value.VerificationProbe );
	}

	void ISceneStartup.OnClientInitialize()
	{
		if ( ClientReadiness != HexRuntimeReadiness.Absent ) return;
		var bootstrapResult = RequireBootstrap();
		if ( bootstrapResult.Failed )
		{
			ClientReadiness = HexRuntimeReadiness.Failed;
			Log.Error( $"HEXAGON_CLIENT_FAILED {bootstrapResult.Error!.Message}" );
			ReportClientBootstrapFailure(
				ClientBootstrapDiagnosticPhase.Bootstrap,
				ClientBootstrapDiagnosticCode.BootstrapUnavailable,
				bootstrapResult.Error.Message );
			return;
		}

		var collector = new SchemaSourceCollector();
		Scene.RunEvent<IHexSchemaSource>( source => source.CollectSchemas( collector ) );
		var descriptor = collector.Require( bootstrapResult.Value.SchemaId );
		if ( descriptor.Failed )
		{
			ClientReadiness = HexRuntimeReadiness.Failed;
			Log.Error( $"HEXAGON_CLIENT_FAILED {descriptor.Error!.Message}" );
			ReportClientBootstrapFailure(
				ClientBootstrapDiagnosticPhase.SchemaDiscovery,
				ClientBootstrapDiagnosticCode.SchemaRuntimeUnavailable,
				descriptor.Error.Message );
			return;
		}

		var clientObject = new GameObject( true, "Hexagon v2 Client" );
		_clientRoot = clientObject.AddComponent<HexClientRootComponent>();
		ClientStore = new HexClientStore
		{
			SubscriberFailureDiagnostic = exception => Log.Warning(
				exception, "HEXAGON_CLIENT_SUBSCRIBER_FAILED store change subscriber threw." )
		};
		var clientNonce = ClientSessionNonce.New();
		ClientStore.PrepareSession( clientNonce );
		ClientTransport = new SandboxClientCommandTransport( Scene, ClientStore, clientNonce );
		_clientController = new HexClientController( ClientTransport );
		_clientRoot.Store = ClientStore;
		_clientRoot.Controller = _clientController;
		try
		{
			descriptor.Value.ConfigureClient?.Invoke(
				new HexClientRuntimeContext( Scene, ClientStore, _clientController ) );
		}
		catch ( Exception exception )
		{
			_clientController.Dispose();
			ClientStore.ClearSession();
			ClientReadiness = HexRuntimeReadiness.Failed;
			Log.Error( exception, "HEXAGON_CLIENT_FAILED schema client configuration threw." );
			ReportClientBootstrapFailure(
				ClientBootstrapDiagnosticPhase.ClientConfiguration,
				ClientBootstrapDiagnosticCode.ClientConfigurationFailed,
				$"{exception.GetType().Name}: {exception.Message}" );
			return;
		}

		ClientReadiness = HexRuntimeReadiness.Ready;
		ClientTransport.PollSession( Stopwatch.GetTimestamp() );
		Log.Info( "HEXAGON_READY client" );
	}

	void Component.INetworkListener.OnActive( Connection connection )
	{
		// Project settings establish the default before admission; reassert the
		// fail-closed policy per remote connection before any identity shell exists.
		if ( !connection.IsHost )
		{
			connection.CanSpawnObjects = false;
			connection.CanRefreshObjects = false;
			connection.CanDestroyObjects = false;
		}
		if ( _sessions.ContainsKey( connection.Id ) ) return;
		// One placement rule answers both this connect and every later embodiment; the game may
		// replace it wholesale. Resolving it here also fails the spawn early, before an object
		// exists, exactly as the previous inline spawn-point scan did.
		var spawn = SpawnSelector.Select( new HexSpawnRequest( connection.Id, null, IsRespawn: false ) );
		if ( spawn.Failed )
		{
			Log.Error(
				$"HEXAGON_PLAYER_SPAWN_FAILED connection={connection.Id} reason=no_spawn_point " +
				$"detail={spawn.Error!.Message}" );
			return;
		}

		GameObject? playerObject = null;
		RuntimePlayerSession<HexPlayerBody>? newSession = null;
		try
		{
			playerObject = new GameObject( true, $"Hexagon Player - {connection.DisplayName}" );
			playerObject.WorldTransform = spawn.Value;
			var player = playerObject.AddComponent<HexPlayerBody>();
			player.HostSpawnSelector = request => SpawnSelector.Select( request );
			// Seat the movement envelope before the body network-spawns, so the owning client receives
			// the geometry its walk mode configures from in the same state as its identity.
			var movementEnvelope = HexagonRuntimeOverrides.ResolveMovementEnvelope();
			player.MovementEnvelope = movementEnvelope;
			// Announce the model and the resolved values. Without this there is no way to tell from a
			// server log WHICH build of the movement code is live, and a stale package served a whole
			// test session against code that had already been replaced.
			Log.Info(
				$"HEXAGON_MOVEMENT_MODEL=windowed-audit-v1 window={movementEnvelope.AuditWindowSeconds}s " +
				$"tolerance={movementEnvelope.HorizontalTolerance} slack={movementEnvelope.InterpolationSlack} " +
				$"angle={movementEnvelope.GroundAngleDegrees} step={movementEnvelope.StepHeight} " +
				$"teleport={movementEnvelope.TeleportGuard} kick={movementEnvelope.ViolationKickSeconds}s" );
			player.HostSetConnection( connection );
			if ( !playerObject.NetworkSpawn( connection ) )
			{
				playerObject.Destroy();
				_spawnSlots.Release( connection.Id );
				Log.Error( $"HEXAGON_PLAYER_SPAWN_FAILED connection={connection.Id} reason=network_spawn_rejected" );
				return;
			}

			newSession = new RuntimePlayerSession<HexPlayerBody>( player, exception =>
				Log.Error( exception, $"Hexagon command cancellation callback failed for connection '{connection.Id}'." ) );
			_sessions.Add( connection.Id, newSession );
			var position = spawn.Value.Position;
			Log.Info( FormattableString.Invariant(
				$"HEXAGON_PLAYER_SPAWNED connection={connection.Id} position={position.x:0.###},{position.y:0.###},{position.z:0.###}" ) );
			if ( HostReadiness == HexRuntimeReadiness.Ready ) _ = newSession.ObserveHostReady();
			// Application-level connection is delayed until the authenticated caller
			// proves possession of its freshly generated client nonce.
		}
		catch ( Exception exception )
		{
			if ( _sessions.Remove( connection.Id, out var registeredSession ) ) registeredSession.Dispose();
			else newSession?.Dispose();
			if ( playerObject is not null && playerObject.IsValid() ) playerObject.Destroy();
			_spawnSlots.Release( connection.Id );
			Log.Error( exception,
				$"HEXAGON_PLAYER_SPAWN_FAILED connection={connection.Id} reason=spawn_exception" );
		}
	}

	void Component.INetworkListener.OnDisconnected( Connection connection )
	{
		_spawnSlots.Release( connection.Id );
		if ( !_sessions.Remove( connection.Id, out var session ) ) return;
		var actor = session.IsApplicationConnected ? BuildActor( connection, session, false ) : (RpcActor?)null;
		session.Disconnect();
		try
		{
			if ( actor is not null ) HostApplication?.Disconnected( actor.Value );
		}
		catch ( Exception exception )
		{
			Log.Error( exception, $"Hexagon disconnect cleanup failed for connection '{connection.Id}'." );
		}
		finally
		{
			try
			{
				// Connection-owned network objects can already be gone by the time the
				// listener is notified. Component.DestroyGameObject() deliberately
				// tolerates that teardown ordering instead of dereferencing a null
				// GameObject during disconnect cleanup.
				session.Player.DestroyGameObject();
			}
			finally
			{
				session.Dispose();
			}
		}
	}

	void Component.INetworkListener.OnBecameHost( Connection previousHost )
	{
		// Hexagon's single-writer persistence model cannot survive a host change: a
		// migrated-to client has no host application, no domain services, and no
		// persistence lease, so accepting hostship would keep an unpersisted zombie
		// session alive under an unprepared, unsandboxed machine's authority. The
		// project networking settings disable migration; this is defense in depth for
		// any composition that re-enables it.
		Log.Error(
			$"HEXAGON_HOST_MIGRATION_REFUSED previous_host={previousHost?.Id.ToString() ?? "unknown"} " +
			"detail=\"Hexagon sessions cannot outlive their prepared host; disconnecting.\"" );
		Sandbox.Networking.Disconnect();
	}

	public bool TryGetPlayer( Guid connectionId, out HexPlayerBody player )
	{
		if ( _sessions.TryGetValue( connectionId, out var session ) )
		{
			player = session.Player;
			return true;
		}
		player = null!;
		return false;
	}

	internal OperationResult<RpcActor> ResolveActor(
		Connection connection,
		ClientSessionScope scope,
		bool requiresStableCharacter )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) ||
			!session.IsApplicationConnected || !session.IsCurrent( scope ) )
			return OperationResult<RpcActor>.Failure( ErrorCode.Unauthorized, "RPC caller has no active host connection binding." );
		var character = HostApplication?.FindActiveCharacter( new ConnectionId( connection.Id ) );
		var lease = session.Boundary.Capture( character?.Id, requiresStableCharacter );
		return OperationResult<RpcActor>.Success( new RpcActor(
			connection,
			new AccountId( connection.SteamId.ValueUnsigned ),
			session.Player,
			character,
			scope,
			lease ) );
	}

	internal CommandAdmissionResult TryBeginCommand( Connection connection, CommandRequestId requestId, int cost ) =>
		_sessions.TryGetValue( connection.Id, out var session )
			? session.TryBeginRequest( requestId, cost )
			: CommandAdmissionResult.Reject( CommandAdmissionFailure.Disconnected );

	internal ClientBootstrapDiagnosticAdmissionResult TryAcceptClientBootstrapDiagnostic(
		Connection connection,
		ClientBootstrapDiagnosticPhase phase,
		ClientBootstrapDiagnosticCode code,
		string? detail ) => _sessions.TryGetValue( connection.Id, out var session )
		? session.TryAcceptBootstrapDiagnostic( phase, code, detail )
		: new ClientBootstrapDiagnosticAdmissionResult(
			false,
			ClientBootstrapDiagnosticAdmissionFailure.Disconnected,
			default );

	internal bool FinishRejectedCommand( Connection connection, CommandRequestId requestId ) =>
		_sessions.TryGetValue( connection.Id, out var session ) && session.FinishRequest( requestId );

	internal OperationResult<ClientSessionScope> BindClientSession(
		Connection connection,
		ClientSessionNonce nonce )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) )
			return OperationResult<ClientSessionScope>.Failure(
				ErrorCode.Conflict, "The host connection session is not ready." );
		if ( !session.TryBind( nonce, out var scope, out _ ) )
			return OperationResult<ClientSessionScope>.Failure(
				ErrorCode.Unauthorized, "The client session nonce cannot be bound to this connection." );
		return OperationResult<ClientSessionScope>.Success( scope );
	}

	internal OperationResult CompleteClientSessionHello(
		Connection connection,
		ClientSessionScope scope )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) || !session.IsCurrent( scope ) )
			return OperationResult.Failure(
				ErrorCode.Unauthorized, "The client session changed before its hello was issued." );
		if ( session.ObserveHelloIssued() && !NotifyConnected( BuildActor( connection, session, false ), session ) )
			return OperationResult.Failure(
				ErrorCode.InternalError, "The host application rejected connection initialization." );
		return OperationResult.Success();
	}

	internal OperationResult<ClientSessionScope> CaptureClientScope( Connection connection )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) || session.Scope is null )
			return OperationResult<ClientSessionScope>.Failure( ErrorCode.Unauthorized, "Client session is not established." );
		return OperationResult<ClientSessionScope>.Success( session.Scope.Value );
	}

	internal bool IsCommandLeaseCurrent( RpcActor actor )
	{
		if ( !_sessions.TryGetValue( actor.Connection.Id, out var session ) ||
			session.Boundary.ConnectionEpoch != actor.ConnectionEpoch )
			return false;
		var character = HostApplication?.FindActiveCharacter( new ConnectionId( actor.Connection.Id ) );
		session.Boundary.ObserveCharacter( character?.Id );
		return session.Boundary.IsCurrent( actor.Session );
	}

	internal CommandCompletionStatus CompleteCommand( RpcActor actor, CommandRequestId requestId )
	{
		if ( !_sessions.TryGetValue( actor.Connection.Id, out var session ) ||
			session.Boundary.ConnectionEpoch != actor.ConnectionEpoch ||
			!session.FinishRequest( requestId ) )
			return CommandCompletionStatus.Disconnected;
		return session.Boundary.IsCurrent( actor.Session )
			? CommandCompletionStatus.Current
			: CommandCompletionStatus.Stale;
	}

	internal OperationResult<ClientStateEpoch> PublishClientStateEpoch(
		Connection connection,
		CharacterId? characterId )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) || !session.IsConnected )
			return OperationResult<ClientStateEpoch>.Failure( ErrorCode.Unauthorized, "Client state recipient is disconnected." );
		return OperationResult<ClientStateEpoch>.Success( session.Boundary.Publish( characterId ) );
	}

	internal OperationResult<ChatDeliveryEpoch> CaptureChatDeliveryEpoch( Connection connection )
	{
		if ( !_sessions.TryGetValue( connection.Id, out var session ) || !session.IsConnected )
			return OperationResult<ChatDeliveryEpoch>.Failure(
				ErrorCode.Unauthorized, "Chat recipient is disconnected." );
		var character = HostApplication?.FindActiveCharacter( new ConnectionId( connection.Id ) );
		session.Boundary.ObserveCharacter( character?.Id );
		return OperationResult<ChatDeliveryEpoch>.Success( new ChatDeliveryEpoch(
			session.Boundary.ConnectionEpoch,
			session.Boundary.CharacterEpoch ) );
	}

	internal bool IsRegisteredPanel( string panelId ) =>
		_hostSchema?.Panels.Contains( panelId ) == true;

	internal int GetSchemaCommandCost( string commandId )
	{
		return SchemaCommandAdmissionCost.Resolve( commandId, registeredId =>
			_hostSchema?.Commands.TryGet( registeredId, out var definition ) == true && definition is not null
				? (int)definition.Cost
				: null );
	}

	internal bool TryStartHostOperation( string name, Func<Task> operation ) =>
		_hostOperations.TryStartTask( name, operation );

	/// <summary>
	/// Explicit awaitable barrier for game-owned scene transition code. s&amp;box's
	/// synchronous GameObjectSystem.Dispose cannot itself await this result.
	/// </summary>
	public ValueTask<OperationResult> ShutdownAsync()
	{
		RequestShutdown();
		return new ValueTask<OperationResult>( BeginCoordinatedShutdown() );
	}

	public override void Dispose()
	{
		RequestShutdown();
		if ( _persistence is not null || HostApplication is not null || _hostInitialization is not null )
			_ = BeginCoordinatedShutdown();
		else
			DisposeHostLifetime();

		_clientController?.Dispose();
		ClientTransport = null;
		ClientStore?.ClearSession();
		ClientReadiness = HexRuntimeReadiness.Disposed;
		foreach ( var session in _sessions.Values ) session.Dispose();
		_sessions.Clear();
		_spawnSlots.Clear();
		base.Dispose();
	}

	private async Task InitializeHostAsync(
		HexSchemaRuntimeDescriptor descriptor,
		CompiledSchema schema,
		SchemaPersistenceBindings persistenceBindings,
		IPersistenceProvider persistence,
		HexHostServicesComponent services,
		Scene scene,
		string persistenceRoot,
		string verificationProbe )
	{
		// Wait for a previous owner of this persistence root (an in-process scene replacement) to
		// finish releasing its lease, then ALWAYS proceed. The predecessor's drain outcome conflates
		// benign engine-teardown faults with real ones, so it is never an authority on whether the
		// store may be opened — the exclusive lease and WAL recovery below decide that. The wait is
		// time-boxed so a stranded drain degrades to a precise lease failure, never an unbounded hang.
		var previousDrain = SceneShutdownBarrier.Previous( persistenceRoot );
		if ( previousDrain is not null )
		{
			if ( await AwaitPredecessorSettlementAsync( previousDrain ) )
			{
				var previousOutcome = await AsyncOperation.Capture(
					() => new ValueTask<OperationResult>( previousDrain ) );
				var verdict = SceneHandoffPolicy.EvaluatePredecessor( previousOutcome );
				if ( verdict.ShouldWarn )
					Log.Warning(
						$"HEXAGON_HANDOFF_DEGRADED root='{persistenceRoot}' state={verdict.State} " +
						$"detail={verdict.Diagnostic}; deferring ownership and integrity to the lease and recovery." );
			}
			else if ( !_disposeRequested )
			{
				Log.Warning(
					$"HEXAGON_HANDOFF_TIMEOUT root='{persistenceRoot}' the previous owner's drain did not settle " +
					$"within {HandoffWaitBudget.TotalSeconds:0}s; proceeding under exclusive-lease protection." );
			}
			SceneShutdownBarrier.Clear( persistenceRoot, previousDrain );
		}

		if ( _disposeRequested )
		{
			_ = await ShutdownResourcesOnceAsync();
			return;
		}

		var recovered = await AsyncOperation.Capture(
			() => persistence.InitializeAsync( _hostLifetime.Token ) );
		if ( !recovered.Succeeded )
		{
			FailHost( $"Persistence recovery failed: {recovered.Exception!.Message}" );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}

		var health = persistence.Health;
		Log.Info(
			$"HEXAGON_RECOVERED schema={schema.Id} sequence={health.Sequence} checkpoint={health.CheckpointSequence} " +
			$"orphan_frames_discarded={health.DiscardedUnacknowledgedFrames} checkpoint_fallback={health.RecoveredFromCheckpointFallback} " +
			$"quarantined={health.RecoveredByQuarantine} root={persistenceRoot}" );
		if ( health.RecoveredByQuarantine )
			Log.Warning(
				$"HEXAGON_PERSISTENCE_QUARANTINE_CONSUMED root={persistenceRoot} quarantine={health.QuarantinePath}; " +
				$"the corrupt store was archived and a fresh store was opened. Re-arm hexagon-persistence-quarantine to quarantine again." );
		if ( _disposeRequested )
		{
			_ = await ShutdownResourcesOnceAsync();
			return;
		}

		var repositories = new DomainRepositories( persistence );
		var invariants = new DomainInvariantValidator(
			repositories,
			schema,
			new SchemaItemShapeCatalog( schema, repositories ),
			descriptor.PersistenceInvariants ).Validate();
		if ( !invariants.IsValid )
		{
			// Report the whole sweep, not just where it stopped: a store with fifty bad rows is
			// fifty restarts if each boot names one. Issue messages never quote persisted values,
			// so no stored string can forge a line here.
			const int reportedIssueLimit = 25;
			foreach ( var issue in invariants.Issues.Take( reportedIssueLimit ) )
				Log.Error( $"HEXAGON_INVARIANT_FAILED path='{issue.Path}' code={issue.Code} detail={issue.Message}" );
			if ( invariants.Issues.Count > reportedIssueLimit )
				Log.Error(
					$"HEXAGON_INVARIANT_FAILED reported={reportedIssueLimit} " +
					$"suppressed={invariants.Issues.Count - reportedIssueLimit} total={invariants.Issues.Count}" );
			var first = invariants.Issues[0];
			FailHost(
				$"Persistence invariant failed at '{first.Path}': {first.Message} " +
				$"({invariants.Issues.Count} issue(s) total)." );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}
		var configuration = new TypedConfigurationStore( persistence, persistenceBindings.PersistenceConfigs );
		var initializedConfiguration = await AsyncOperation.Capture(
			() => configuration.InitializeAsync( _hostLifetime.Token ) );
		if ( !initializedConfiguration.Succeeded )
		{
			FailHost( $"Typed configuration startup failed: {initializedConfiguration.Exception!.Message}" );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}
		if ( initializedConfiguration.Value.Failed )
		{
			FailHost( $"Typed configuration startup failed: {initializedConfiguration.Value.Error!.Message}" );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}
		var context = new HexHostRuntimeContext(
			scene,
			schema,
			persistence,
			repositories,
			configuration,
			services,
			persistenceRoot,
			verificationProbe );
		var created = await AsyncOperation.Capture(
			() => ValueTask.FromResult( descriptor.CreateHostApplication( context ) ) );
		if ( !created.Succeeded || created.Value is null )
		{
			FailHost( $"Host application construction failed: {created.Exception?.Message ?? "no application was returned"}." );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}

		HostApplication = created.Value;
		var initialized = await AsyncOperation.Capture(
			() => HostApplication.InitializeAsync( _hostLifetime.Token ) );
		if ( !initialized.Succeeded )
		{
			FailHost( $"Host application initialization threw: {initialized.Exception!.Message}" );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}
		if ( initialized.Value.Failed )
		{
			FailHost( initialized.Value.Error!.Message );
			_ = await ShutdownResourcesOnceAsync();
			return;
		}
		if ( _disposeRequested )
		{
			_ = await ShutdownResourcesOnceAsync();
			return;
		}

		HostReadiness = HexRuntimeReadiness.Ready;
		foreach ( var session in _sessions.Values )
		{
			if ( session.ObserveHostReady() && session.Player.HostConnection is not null )
				_ = NotifyConnected( BuildActor( session.Player.HostConnection, session, false ), session );
		}
		Log.Info( $"HEXAGON_READY host schema={schema.Id} sequence={persistence.Health.Sequence} root={persistenceRoot}" );
	}

	private static readonly TimeSpan HandoffWaitBudget = TimeSpan.FromSeconds( 10 );

	/// <summary>
	/// Waits for a previous owner of the same persistence root to finish draining, bounded so a
	/// stranded drain cannot hang host initialization forever. Returns true if the predecessor
	/// settled within the budget, false on timeout or host disposal. Built only from async
	/// primitives proven under the s&amp;box whitelist elsewhere in this assembly — a
	/// CancellationTokenSource timer, token registrations, a TaskCompletionSource, and
	/// AsyncOperation.Capture — with no Task.Delay/WhenAny/WaitAsync.
	/// </summary>
	private async ValueTask<bool> AwaitPredecessorSettlementAsync( Task<OperationResult> previousDrain )
	{
		if ( previousDrain.IsCompleted ) return true;
		var settlement = new TaskCompletionSource<bool>( TaskCreationOptions.RunContinuationsAsynchronously );
		using var timeout = new CancellationTokenSource();
		using var timeoutRegistration = timeout.Token.Register(
			static state => ((TaskCompletionSource<bool>)state!).TrySetResult( false ), settlement );
		using var lifetimeRegistration = _hostLifetime.Token.Register(
			static state => ((TaskCompletionSource<bool>)state!).TrySetResult( false ), settlement );
		_ = previousDrain.ContinueWith( _ => settlement.TrySetResult( true ) );
		timeout.CancelAfter( HandoffWaitBudget );
		var settled = await AsyncOperation.Capture( () => new ValueTask<bool>( settlement.Task ) );
		return settled.Succeeded && settled.Value;
	}

	private void RequestShutdown()
	{
		if ( _disposeRequested ) return;
		_disposeRequested = true;
		_hostOperations.StopAdmission();
		if ( !_hostLifetimeDisposed )
		{
			try
			{
				_hostLifetime.Cancel();
			}
			catch ( Exception exception )
			{
				Log.Error( exception, "Hexagon host lifetime cancellation callback failed." );
			}
		}
		if ( HostReadiness is not (HexRuntimeReadiness.Absent or HexRuntimeReadiness.Disposed) )
			HostReadiness = HexRuntimeReadiness.Disposing;
	}

	private Task<OperationResult> BeginCoordinatedShutdown()
	{
		if ( _hostShutdown is not null ) return _hostShutdown;
		_hostShutdown = CoordinateShutdownAsync( _hostInitialization );
		if ( !string.IsNullOrWhiteSpace( _persistenceRoot ) )
			SceneShutdownBarrier.Publish( _persistenceRoot, _hostShutdown );
		_ = _hostShutdown.ContinueWith( static task =>
		{
			if ( task.IsFaulted )
				Log.Error( $"HEXAGON_DRAIN_FAILED {task.Exception?.GetBaseException().Message}" );
		} );
		return _hostShutdown;
	}

	private async Task<OperationResult> CoordinateShutdownAsync( Task? initialization )
	{
		if ( initialization is not null )
			_ = await AsyncOperation.Capture( () => new ValueTask( initialization ) );
		return await ShutdownResourcesOnceAsync();
	}

	private Task<OperationResult> ShutdownResourcesOnceAsync() =>
		_resourceShutdown ??= ShutdownHostAsync();

	private async Task<OperationResult> ShutdownHostAsync()
	{
		Exception? firstFailure = null;
		PersistenceShutdownResult? persistenceShutdown = null;
		var application = HostApplication;
		var disconnects = new List<(RuntimePlayerSession<HexPlayerBody> Session, RpcActor? Actor)>();
		foreach ( var session in _sessions.Values )
		{
			RpcActor? actor = null;
			if ( session.Player.HostConnection is Connection connection && session.Scope is not null )
			{
				try { actor = BuildActor( connection, session, false ); }
				catch ( Exception exception )
				{
					firstFailure ??= exception;
					Log.Error( exception, $"Could not capture shutdown actor for '{connection.Id}'." );
				}
			}
			disconnects.Add( (session, actor) );
		}

		foreach ( var disconnect in disconnects )
		{
			disconnect.Session.Disconnect();
			if ( application is not null && disconnect.Actor is RpcActor actor )
			{
				try { application.Disconnected( actor ); }
				catch ( Exception exception )
				{
					firstFailure ??= exception;
					Log.Error( exception, $"Host application disconnect failed for '{actor.Connection.Id}'." );
				}
			}
		}

		var commandDrain = await _hostOperations.DrainAsync();
		foreach ( var failure in commandDrain.Failures )
		{
			if ( !failure.WasCanceled ) firstFailure ??= failure.Exception;
			Log.Warning(
				$"HEXAGON_HOST_OPERATION_FAILED id={failure.OperationId} name={failure.Name} " +
				$"canceled={failure.WasCanceled} message={failure.Exception.Message}" );
		}

		var hostServices = _hostServices;
		_hostServices = null;
		if ( hostServices is not null )
		{
			try
			{
				hostServices.Runtime = null;
				if ( hostServices.GameObject.IsValid() ) hostServices.GameObject.Destroy();
			}
			catch ( Exception exception ) { firstFailure ??= exception; }
		}

		HostApplication = null;
		if ( application is not null )
		{
			var disposedApplication = await AsyncOperation.Capture( application.DisposeAsync );
			if ( !disposedApplication.Succeeded ) firstFailure ??= disposedApplication.Exception;
		}

		var persistence = _persistence;
		_persistence = null;
		_hostSchema = null;
		if ( persistence is not null )
		{
			var drained = await AsyncOperation.Capture( () => persistence.ShutdownAsync() );
			if ( !drained.Succeeded ) firstFailure ??= drained.Exception;
			else
			{
				var shutdown = drained.Value!;
				persistenceShutdown = shutdown;
				if ( !shutdown.IsRecoverable )
					firstFailure ??= new InvalidOperationException(
						shutdown.Detail ?? "Persistence shutdown is not recoverable." );
			}
			var disposedPersistence = await AsyncOperation.Capture( persistence.DisposeAsync );
			if ( !disposedPersistence.Succeeded ) firstFailure ??= disposedPersistence.Exception;
		}

		try { DisposeHostLifetime(); }
		catch ( Exception exception ) { firstFailure ??= exception; }

		if ( firstFailure is null && application is not null && persistenceShutdown?.IsClean == true )
		{
			try
			{
				var finalized = application.CompleteQuiescedShutdown( persistenceShutdown );
				if ( finalized.Failed )
					firstFailure = new InvalidOperationException( finalized.Error!.Message );
			}
			catch ( Exception exception ) { firstFailure = exception; }
		}

		if ( firstFailure is not null )
		{
			HostReadiness = HexRuntimeReadiness.Failed;
			Log.Error( firstFailure, "HEXAGON_DRAIN_FAILED host resources did not close cleanly." );
			return OperationResult.Failure( ErrorCode.InternalError, "Host resources did not close cleanly." );
		}

		if ( HostReadiness != HexRuntimeReadiness.Failed ) HostReadiness = HexRuntimeReadiness.Disposed;
		if ( persistenceShutdown is not null && persistenceShutdown.IsClean )
		{
			Log.Info( "HEXAGON_DRAINED host" );
		}
		else if ( persistenceShutdown is not null )
		{
			Log.Warning(
				$"HEXAGON_DRAIN_DEGRADED durable_sequence={persistenceShutdown.DurableSequence} " +
				$"checkpoint_sequence={persistenceShutdown.CheckpointSequence} " +
				$"lease_released={persistenceShutdown.LeaseReleased} detail={persistenceShutdown.Detail}" );
		}
		else
		{
			Log.Warning( "HEXAGON_DRAIN_DEGRADED persistence_shutdown=unavailable" );
		}
		return OperationResult.Success();
	}

	private OperationResult<HexHostServicesComponent> CreateHostServices()
	{
		GameObject? servicesObject = null;
		try
		{
			servicesObject = new GameObject( true, "Hexagon v2 Host Services" );
			var services = servicesObject.AddComponent<HexHostServicesComponent>();
			services.Runtime = this;
			return HostServicePublication.RequirePublished(
				services,
				servicesObject.NetworkSpawn,
				() =>
				{
					services.Runtime = null;
					if ( servicesObject.IsValid() ) servicesObject.Destroy();
				} );
		}
		catch ( Exception exception )
		{
			try { if ( servicesObject is not null && servicesObject.IsValid() ) servicesObject.Destroy(); }
			catch ( Exception ) { }
			return OperationResult<HexHostServicesComponent>.Failure(
				ErrorCode.InternalError,
				$"Hexagon host services could not be created: {exception.Message}" );
		}
	}

	private void ReportClientBootstrapFailure(
		ClientBootstrapDiagnosticPhase phase,
		ClientBootstrapDiagnosticCode code,
		string detail )
	{
		try
		{
			var services = _runtimeScene.GetAll<HexHostServicesComponent>().FirstOrDefault();
			if ( services is null ) return;
			services.ReportClientBootstrapDiagnostic(
				phase,
				code,
				ClientBootstrapDiagnosticContract.PrepareDetail( code, detail ) );
		}
		catch ( Exception exception )
		{
			Log.Warning( exception, "Hexagon could not report the bounded client bootstrap diagnostic to the host." );
		}
	}

	private OperationResult<HexagonBootstrapComponent> RequireBootstrap()
	{
		var bootstraps = Scene.GetAll<HexagonBootstrapComponent>().ToArray();
		if ( bootstraps.Length != 1 )
			return OperationResult<HexagonBootstrapComponent>.Failure(
				ErrorCode.SchemaInvalid, $"Expected exactly one Hexagon bootstrap, found {bootstraps.Length}." );
		if ( string.IsNullOrWhiteSpace( bootstraps[0].SchemaId ) )
			return OperationResult<HexagonBootstrapComponent>.Failure(
				ErrorCode.SchemaInvalid, "Hexagon bootstrap schema ID is blank." );
		try
		{
			if ( !string.IsNullOrWhiteSpace( bootstraps[0].PersistenceRootOverride ) )
				_ = PrefixedPersistenceStorage.Normalize( bootstraps[0].PersistenceRootOverride );
		}
		catch ( Exception exception )
		{
			return OperationResult<HexagonBootstrapComponent>.Failure( ErrorCode.ConfigurationInvalid, exception.Message );
		}
		return OperationResult<HexagonBootstrapComponent>.Success( bootstraps[0] );
	}

	private static OperationResult<HostBootstrapOptions> ResolveHostOptions( HexagonBootstrapComponent bootstrap ) =>
		HostBootstrapResolution.Resolve(
			HexagonRuntimeOverrides.PersistenceRoot,
			bootstrap.PersistenceRootOverride,
			HexagonRuntimeOverrides.VerificationProbe,
			bootstrap.VerificationProbe,
			PrefixedPersistenceStorage.Normalize );

	private RpcActor BuildActor( Connection connection, RuntimePlayerSession<HexPlayerBody> session, bool requiresStableCharacter )
	{
		var character = HostApplication?.FindActiveCharacter( new ConnectionId( connection.Id ) );
		return new RpcActor(
			connection,
			new AccountId( connection.SteamId.ValueUnsigned ),
			session.Player,
			character,
			session.Scope ?? throw new InvalidOperationException( "Client session is not established." ),
			session.Boundary.Capture( character?.Id, requiresStableCharacter ) );
	}

	private bool NotifyConnected( RpcActor actor, RuntimePlayerSession<HexPlayerBody> session )
	{
		var succeeded = false;
		try
		{
			var application = HostApplication ??
				throw new InvalidOperationException( "The host application is not initialized." );
			application.Connected( actor );
			succeeded = true;
			Log.Info(
				$"HEXAGON_SESSION_BOUND account={actor.AccountId.Value} connection={actor.Connection.Id} " +
				$"epoch={actor.ClientScope.Connection}" );
			return true;
		}
		catch ( Exception exception )
		{
			Log.Error( exception, $"Hexagon connection initialization failed for '{actor.Connection.Id}'." );
			return false;
		}
		finally
		{
			if ( !session.CompleteApplicationConnection( succeeded ) || !succeeded )
				session.Disconnect();
		}
	}

	private void PollLifecycle()
	{
		if ( ClientReadiness == HexRuntimeReadiness.Ready )
			ClientTransport?.PollSession( Stopwatch.GetTimestamp() );
		if ( _hostInitialization?.IsFaulted == true && !_hostFailureLogged )
		{
			_hostFailureLogged = true;
			FailHost( _hostInitialization.Exception?.GetBaseException().Message ?? "Host initialization faulted." );
			_ = BeginCoordinatedShutdown();
		}
	}

	private void FailHost( string message )
	{
		HostReadiness = HexRuntimeReadiness.Failed;
		Log.Error( $"HEXAGON_HOST_FAILED {message}" );
	}

	private void DisposeHostLifetime()
	{
		if ( _hostLifetimeDisposed ) return;
		_hostLifetimeDisposed = true;
		_hostLifetime.Dispose();
	}

}