Framework/Scripting/LuaContent.cs

The Lua script transfer layer for the framework. The host builds a SHA-256 manifest of every shared and client Lua file (framework plus gamemode/addon chain, server files excluded) and streams requested files to joining clients in 16K chunks. Clients re-verify each file's hash into a memory filesystem, cache copies in a per-server temp folder under the data dir, then load their realm from exactly that verified set.

File AccessNetworking
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Serialization;
using RoleplayFramework.Core;
using RoleplayFramework.Diagnostics;
using RoleplayFramework.Players;

namespace RoleplayFramework.Scripting;

public sealed class ContentFile
{
	[JsonPropertyName( "p" )] public string Path { get; set; }
	[JsonPropertyName( "h" )] public string Hash { get; set; }
	[JsonPropertyName( "s" )] public int Size { get; set; }
}

/// <summary>
/// Script distribution. The server lists every shared and client file it runs (framework + gamemode chain)
/// with a SHA-256 hash; server (sv_) files are never listed and can never be requested.
///
/// A joining client:
/// 1. receives the manifest,
/// 2. reuses any local file whose hash matches,
/// 3. requests the rest, which the server streams in bounded chunks,
/// 4. verifies every file's hash, keeps them in memory only, and loads its realm from exactly that verified set.
///
/// Limits: file count, per-file size, total size, chunk size, and a session timeout enforced by the host.
/// </summary>
public static class LuaContent
{
	public const int MaxFiles = 2000;
	public const int MaxManifestChars = 256 * 1024;
	public const int MaxFileChars = 512 * 1024;
	public const int MaxTotalChars = 8 * 1024 * 1024;
	public const int ChunkChars = 16 * 1024;

	public static string HashText( string text ) =>
		Convert.ToHexString( System.Security.Cryptography.SHA256.HashData( Encoding.UTF8.GetBytes( text ) ) );

	/// <summary>Reads the shared/client file set for a gamemode. Returns the manifest plus a snapshot of the file texts.</summary>
	public static List<ContentFile> Build( BaseFileSystem fs, GamemodeRegistry registry, string gamemodeId, Dictionary<string, string> snapshot = null, IEnumerable<AddonManifest> addons = null )
	{
		var paths = new List<string>();

		if ( fs.DirectoryExists( LuaRealm.FrameworkRoot ) )
			paths.AddRange( fs.FindFile( LuaRealm.FrameworkRoot, "*.lua", false ).Select( f => $"{LuaRealm.FrameworkRoot}/{f}" ) );

		foreach ( var m in registry.LoadOrder( gamemodeId ) ?? new List<GamemodeManifest>() )
		{
			paths.Add( $"{m.Root}/gamemode.json" );
			paths.AddRange( fs.FindFile( m.Root, "*.lua", true ).Select( f => $"{m.Root}/{f}" ) );
		}

		// Active addons only: a switched-off or incompatible addon is never sent, so clients load exactly the server's set.
		foreach ( var a in addons ?? Enumerable.Empty<AddonManifest>() )
		{
			paths.Add( $"{a.Root}/addon.json" );
			paths.AddRange( fs.FindFile( a.Root, "*.lua", true ).Select( f => $"{a.Root}/{f}" ) );
		}

		var result = new List<ContentFile>();
		foreach ( var p in paths
			.Where( p => p.EndsWith( "/gamemode.json" ) || p.EndsWith( "/addon.json" ) || LuaPaths.RealmOf( p ) is LuaRealmKind.Shared or LuaRealmKind.Client )
			.Distinct()
			.OrderBy( p => p, StringComparer.Ordinal ) )
		{
			var text = fs.ReadAllText( p );
			if ( text.Length > MaxFileChars ) throw new InvalidOperationException( $"{p} is larger than {MaxFileChars} characters" );
			snapshot?.Add( p, text );
			result.Add( new ContentFile { Path = p, Hash = HashText( text ), Size = text.Length } );
		}

		if ( result.Sum( f => (long)f.Size ) > MaxTotalChars ) throw new InvalidOperationException( "shared/client scripts exceed the total size limit" );
		return result;
	}

	/// <summary>Checks one untrusted manifest entry. Returns null if acceptable.</summary>
	static string CheckEntry( ContentFile f )
	{
		if ( f?.Path is null || f.Hash is null || f.Hash.Length != 64 ) return "malformed manifest entry";
		if ( !f.Path.StartsWith( "lua/" ) || f.Path.Contains( ".." ) || f.Path.Contains( '\\' ) || f.Path.Contains( ':' ) || f.Path.Length > 200 )
			return $"manifest path not allowed: {f.Path}";
		if ( LuaPaths.RealmOf( f.Path ) == LuaRealmKind.Server )
			return $"manifest lists a server-only file ({f.Path}); refusing";
		if ( !f.Path.EndsWith( ".lua" ) && !f.Path.EndsWith( "/gamemode.json" ) && !(f.Path.StartsWith( AddonRegistry.AddonsRoot + "/" ) && f.Path.EndsWith( "/addon.json" )) )
			return $"manifest lists a non-script file ({f.Path})";
		if ( f.Size < 0 || f.Size > MaxFileChars ) return $"manifest file too large ({f.Path})";
		return null;
	}

	/// <summary>Verifies an untrusted manifest against a filesystem. Returns null when everything matches, else the problem.</summary>
	public static string Verify( BaseFileSystem fs, List<ContentFile> files )
	{
		if ( files is null ) return "manifest missing";
		if ( files.Count > MaxFiles ) return $"manifest lists too many files ({files.Count})";

		var problems = new List<string>();
		foreach ( var f in files )
		{
			if ( CheckEntry( f ) is { } bad ) return bad;
			if ( !fs.FileExists( f.Path ) ) problems.Add( $"missing {f.Path}" );
			else if ( !string.Equals( HashText( fs.ReadAllText( f.Path ) ), f.Hash, StringComparison.OrdinalIgnoreCase ) ) problems.Add( $"different {f.Path}" );
		}

		if ( problems.Count == 0 ) return null;
		return $"{problems.Count} script file(s) differ from the server: {string.Join( ", ", problems.Take( 3 ) )}{(problems.Count > 3 ? ", ..." : "")}";
	}

	// ------------------------------------------------------------ host side

	/// <summary>Host: send the manifest to a client that passed Framework.Hello.</summary>
	internal static void SendManifestTo( Connection c )
	{
		var json = LuaRuntime.ManifestJson;
		var gm = LuaRuntime.Server?.Gamemode?.Id;
		if ( json is null || gm is null )
		{
			PlayerSessions.Reject( c, PlayerSessions.Get( c.Id ), "The server has no gamemode running" );
			return;
		}

		using ( Rpc.FilterInclude( c ) ) Manifest( gm, json, string.Join( ",", Core.ContentPackages.Required ) );
	}

	/// <summary>Client → host: send me these manifest entries (by index).</summary>
	[Rpc.Host]
	public static void RequestFiles( string indexList )
	{
		var caller = Rpc.Caller;
		if ( !PlayerSessions.Guard( caller, "Framework.RequestFiles", 2, 0.1, out var session, requireReady: false ) ) return;

		if ( session.State != SessionState.AwaitingContent )
		{
			PlayerSessions.ReportViolation( caller, session, "Framework.RequestFiles", $"unexpected in state {session.State}" );
			return;
		}

		var files = LuaRuntime.ManifestFiles;
		var texts = LuaRuntime.ManifestTexts;
		if ( files is null || texts is null ) return;

		var indices = new List<int>();
		foreach ( var part in (indexList ?? "").Split( ',', StringSplitOptions.RemoveEmptyEntries ).Take( MaxFiles + 1 ) )
		{
			if ( !int.TryParse( part, out var i ) || i < 0 || i >= files.Count || indices.Contains( i ) )
			{
				PlayerSessions.Reject( caller, session, "Invalid script file request" );
				return;
			}
			indices.Add( i );
		}

		long total = 0;
		int chunks = 0;
		using ( Rpc.FilterInclude( caller ) )
		{
			foreach ( var i in indices )
			{
				var text = texts[files[i].Path];
				total += text.Length;
				int count = Math.Max( 1, (text.Length + ChunkChars - 1) / ChunkChars );
				for ( int k = 0; k < count; k++ )
				{
					int start = k * ChunkChars;
					FileChunk( i, k, count, text.Substring( start, Math.Min( ChunkChars, text.Length - start ) ) );
					chunks++;
				}
			}
		}

		RpLog.Info( "content", $"Sent {indices.Count} script file(s) ({total:N0} chars, {chunks} chunks) to {session.DisplayName}" );
	}

	// ------------------------------------------------------------ client side

	sealed class Download
	{
		public string Gamemode;
		public List<ContentFile> Files;
		public BaseFileSystem Fs;
		public Dictionary<int, string[]> Pending = new();
		public long ReceivedChars;
		public long TotalMissingChars;
	}

	static Download _download;

	/// <summary>Host → one client: the content it must have. Runs on the client.</summary>
	[Rpc.Broadcast( NetFlags.HostOnly )]
	public static void Manifest( string gamemodeId, string json, string packages )
	{
		_pendingPackages = Core.ContentPackages.ParseFromServer( packages );
		if ( Networking.IsHost ) return; // the host already runs its own client realm
		_download = null;

		string error = null;
		List<ContentFile> files = null;

		if ( !FrameworkConfig.IsValidId( gamemodeId ) ) error = "server sent an invalid gamemode id";
		else if ( json is null || json.Length > MaxManifestChars ) error = "server manifest too large";
		else if ( !Json.TryDeserialize( json, out files ) || files is null ) error = "server manifest could not be read";
		else if ( files.Count > MaxFiles ) error = $"server manifest lists too many files ({files.Count})";
		else if ( files.Sum( f => (long)(f?.Size ?? 0) ) > MaxTotalChars ) error = "server scripts exceed the total size limit";
		else error = files.Select( CheckEntry ).FirstOrDefault( e => e is not null );

		if ( error is not null ) { Finish( false, error ); return; }

		// Build the client's script set in memory: matching local files are reused, the rest is downloaded.
		var d = new Download { Gamemode = gamemodeId, Files = files, Fs = FileSystem.CreateMemoryFileSystem() };
		var missing = new List<int>();
		BeginServerFolder( gamemodeId );

		// Reuse what this client already has for THIS server (its temp folder from earlier joins), then the game's own
		// files. Anything whose fingerprint doesn't match the server's is downloaded again.
		for ( int i = 0; i < files.Count; i++ )
		{
			var f = files[i];
			var text = ReadIfMatches( FileSystem.Data, CachePath( f.Path ), f.Hash ) ?? ReadIfMatches( FileSystem.Mounted, f.Path, f.Hash );
			if ( text is not null )
			{
				Store( d.Fs, f.Path, text );
				SaveToCache( f.Path, text );
				continue;
			}
			missing.Add( i );
		}

		_download = d;
		Core.LoadingProgress.SetInfo( string.IsNullOrWhiteSpace( Networking.ServerName ) ? null : Networking.ServerName.Trim(), gamemodeId );
		Core.LoadingProgress.Phase( "Downloading scripts", 0.05f, 0.35f, missing.Count == 0 ? "all files already here" : $"{missing.Count} of {files.Count} file(s)" );
		d.TotalMissingChars = missing.Sum( i => (long)Math.Max( 1, files[i].Size ) );
		RpLog.Info( "content", $"Manifest for '{gamemodeId}': {files.Count} files, {files.Count - missing.Count} already present, {missing.Count} to download (server temp folder: {FileSystem.Data.GetFullPath( _serverFolder )})" );

		if ( missing.Count == 0 ) { LoadDownloaded(); return; }

		foreach ( var i in missing ) d.Pending[i] = null;
		RequestFiles( string.Join( ",", missing ) );
	}

	/// <summary>Host → one client: part of a requested script file.</summary>
	[Rpc.Broadcast( NetFlags.HostOnly )]
	public static void FileChunk( int index, int chunk, int chunkCount, string data )
	{
		if ( Networking.IsHost ) return;
		var d = _download;
		if ( d is null || !d.Pending.ContainsKey( index ) ) return; // not requested: ignore

		var file = d.Files[index];
		int expectedChunks = Math.Max( 1, (file.Size + ChunkChars - 1) / ChunkChars );
		if ( chunkCount != expectedChunks || chunk < 0 || chunk >= chunkCount || data is null || data.Length > ChunkChars )
		{
			Finish( false, $"bad data received for {file.Path}" );
			return;
		}

		var parts = d.Pending[index] ??= new string[chunkCount];
		if ( parts[chunk] is not null ) return; // duplicate
		parts[chunk] = data;
		d.ReceivedChars += data.Length;
		Core.LoadingProgress.Step( d.TotalMissingChars > 0 ? (float)d.ReceivedChars / d.TotalMissingChars : 1f,
			$"{file.Path}  ({d.Files.Count - d.Pending.Count + 1}/{d.Files.Count})" );
		if ( d.ReceivedChars > MaxTotalChars ) { Finish( false, "server sent more data than its manifest declared" ); return; }

		if ( parts.Any( p => p is null ) ) return;

		var text = string.Concat( parts );
		if ( !string.Equals( HashText( text ), file.Hash, StringComparison.OrdinalIgnoreCase ) )
		{
			Finish( false, $"downloaded {file.Path} does not match its hash" );
			return;
		}

		Store( d.Fs, file.Path, text );
		SaveToCache( file.Path, text );
		d.Pending.Remove( index );

		if ( d.Pending.Count == 0 )
		{
			RpLog.Info( "content", $"Downloaded and verified {d.Files.Count} script file(s) ({d.ReceivedChars:N0} chars)" );
			LoadDownloaded();
		}
	}

	/// <summary>
	/// Client-side temp folders for server scripts: one per server, <c>data/…/roleplay/tmp/servers/&lt;server-id&gt;/lua/...</c>.
	/// Servers may run their own modified Lua, so each server's files are kept apart (never mixed with another
	/// server's or with this player's own editable gamemodes in <c>roleplay/lua</c>). The folders are only a cache:
	/// every join re-checks every file's fingerprint against the server.
	/// </summary>
	public const string TempServersRoot = "roleplay/tmp/servers";
	public const int MaxServerFolders = 20;
	public const int ServerFolderMaxAgeDays = 30;

	static string _serverFolder;

	static string CachePath( string manifestPath ) => $"{_serverFolder}/{manifestPath}";

	/// <summary>Stable id of the server we are joining: the host's Steam id (dedicated servers have their own).</summary>
	internal static string ServerKey()
	{
		var host = Connection.Host;
		var id = host?.SteamId.ToString();
		if ( string.IsNullOrEmpty( id ) || id == "0" ) id = "unknown";
		return new string( id.Where( char.IsAsciiLetterOrDigit ).Take( 32 ).ToArray() );
	}

	static void BeginServerFolder( string gamemodeId )
	{
		_serverFolder = $"{TempServersRoot}/{ServerKey()}";
		var fs = FileSystem.Data;
		if ( !fs.DirectoryExists( _serverFolder ) ) fs.CreateDirectory( _serverFolder );
		fs.WriteAllText( $"{_serverFolder}/server.txt",
			$"name={Net.FrameworkRpc.Bound( Networking.ServerName, 64 )}\ngamemode={gamemodeId}\nlastUsed={DateTimeOffset.UtcNow.ToUnixTimeSeconds()}\n" );
		PruneServerFolders();
	}

	/// <summary>Keeps the temp folder small: drops folders unused for 30 days, and the oldest beyond 20 servers.</summary>
	static void PruneServerFolders()
	{
		try
		{
			var fs = FileSystem.Data;
			var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
			var folders = fs.FindDirectory( TempServersRoot, "*", false )
				.Select( d =>
				{
					long used = 0;
					var info = $"{TempServersRoot}/{d}/server.txt";
					if ( fs.FileExists( info ) )
						foreach ( var line in fs.ReadAllText( info ).Split( '\n' ) )
							if ( line.StartsWith( "lastUsed=" ) ) long.TryParse( line.Substring( 9 ), out used );
					return (Dir: d, Used: used);
				} )
				.OrderByDescending( x => x.Used )
				.ToList();

			for ( int i = 0; i < folders.Count; i++ )
			{
				var (dir, used) = folders[i];
				var path = $"{TempServersRoot}/{dir}";
				if ( path == _serverFolder ) continue;
				if ( i >= MaxServerFolders || now - used > ServerFolderMaxAgeDays * 86400L )
					fs.DeleteDirectory( path, true );
			}
		}
		catch ( Exception e )
		{
			RpLog.Warn( "content", $"Could not tidy the server temp folders: {e.Message}" );
		}
	}

	static string ReadIfMatches( BaseFileSystem fs, string path, string hash )
	{
		try
		{
			if ( !fs.FileExists( path ) ) return null;
			var text = fs.ReadAllText( path );
			return string.Equals( HashText( text ), hash, StringComparison.OrdinalIgnoreCase ) ? text : null;
		}
		catch ( Exception ) { return null; }
	}

	/// <summary>
	/// Writes a verified script into this server's temp folder, so it is visible and reused on the next join to the
	/// same server. Editing these copies can't change what runs: every join re-checks fingerprints.
	/// </summary>
	static void SaveToCache( string manifestPath, string text )
	{
		try
		{
			var path = CachePath( manifestPath );
			if ( ReadIfMatches( FileSystem.Data, path, HashText( text ) ) is not null ) return;
			Store( FileSystem.Data, path, text );
		}
		catch ( Exception e )
		{
			RpLog.Warn( "content", $"Could not save {manifestPath} to the download folder: {e.Message}" );
		}
	}

	static void Store( BaseFileSystem fs, string path, string text )
	{
		int slash = path.LastIndexOf( '/' );
		if ( slash > 0 ) fs.CreateDirectory( path.Substring( 0, slash ) );
		fs.WriteAllText( path, text );
	}

	static void LoadDownloaded() => _ = LoadDownloadedAsync();

	static List<string> _pendingPackages = new();

	static async System.Threading.Tasks.Task LoadDownloadedAsync()
	{
		var d = _download;

		// The server's required cloud content, before its scripts run (they may reference these models/materials).
		if ( _pendingPackages.Count > 0 )
		{
			RpLog.Info( "content", $"Server requires {_pendingPackages.Count} package(s): {string.Join( ", ", _pendingPackages )}" );
			Core.LoadingProgress.Phase( "Mounting content", 0.35f, 0.8f, $"{_pendingPackages.Count} package(s)" );
			var failed = await Core.ContentPackages.MountAll( _pendingPackages,
				( i, n, id ) => { if ( _download == d ) Core.LoadingProgress.Step( (float)i / Math.Max( 1, n ), $"{Core.ContentPackages.TitleOrIdent( id )}  ({i + 1}/{n})" ); } );
			if ( failed.Count > 0 ) RpLog.Warn( "content", $"Could not mount: {string.Join( ", ", failed )} (continuing without them)" );
			if ( _download != d ) return;
		}

		// Don't report ready (and get spawned) before the world's map exists on this machine.
		Core.LoadingProgress.Phase( "Loading map", 0.8f, 0.9f );
		var mapError = await WorldMap.WaitForSceneMap( Game.ActiveScene );
		if ( mapError is not null ) { Finish( false, mapError ); return; }
		if ( _download != d ) return; // superseded

		Core.LoadingProgress.Phase( "Loading client scripts", 0.9f, 0.98f, d.Gamemode );
		await Core.LoadingProgress.Show();
		if ( _download != d ) return;
		var error = Verify( d.Fs, d.Files );
		if ( error is null && !LuaRuntime.StartClient( d.Gamemode, d.Fs, out var loadError, fromServer: true ) )
			error = $"client scripts failed to load: {loadError}";
		Finish( error is null, error );
	}

	static void Finish( bool ok, string error )
	{
		_download = null;
		// Success: the screen stays until our body is in the world (RpPlayer), at most a few seconds.
		if ( ok ) { Core.LoadingProgress.Phase( "Spawning", 0.98f, 1f ); Core.LoadingProgress.Step( 1f, "" ); Core.LoadingProgress.EndAfter( 8 ); }
		else Core.LoadingProgress.End();
		if ( !ok ) RpLog.Error( "content", error );
		Ready( ok, error ?? "" );
	}

	/// <summary>Client → host: content verified and client realm loaded (or why not).</summary>
	[Rpc.Host]
	public static void Ready( bool ok, string error )
	{
		var caller = Rpc.Caller;
		if ( !PlayerSessions.Guard( caller, "Framework.ContentReady", 2, 0.2, out var session, requireReady: false ) ) return;

		if ( session.State != SessionState.AwaitingContent )
		{
			PlayerSessions.ReportViolation( caller, session, "Framework.ContentReady", $"unexpected in state {session.State}" );
			return;
		}

		if ( !ok )
		{
			PlayerSessions.Reject( caller, session, $"Could not load the server's scripts: {Net.FrameworkRpc.Bound( error, 300 )}" );
			return;
		}

		session.State = SessionState.Ready;
		RpLog.Info( "content", $"{session.DisplayName} verified content and loaded client scripts" );
		PlayerManager.Current?.OnSessionReady( caller );
	}
}