Framework/Core/ContentPackages.cs

Static content package manager for the server. Collects required package idents from framework config, each gamemode in the chain and addon manifests, plus JSON collections loaded over HTTP or from a file, then fetches and mounts them through Package.FetchAsync/MountAsync. Also queues runtime additions from Lua (with a pump task, batch events and a Busy/WhenIdle wait) and parses the server's package list on clients.

Http Calls
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using RoleplayFramework.Diagnostics;

namespace RoleplayFramework.Core;

/// <summary>
/// Cloud content a server requires: sbox.game packages (models, materials, sounds, maps, ...; packages published through
/// the Steam Workshop are packages too). Sources, merged and de-duplicated:
/// <list type="bullet">
/// <item>server config <c>"Packages"</c> and <c>"Collections"</c> (framework.json)</item>
/// <item>the gamemode chain's <c>"packages"</c> and <c>"collections"</c> (gamemode.json)</item>
/// <item>Lua: <c>content.AddPackage</c> / <c>content.AddCollection</c>, at load time or while the server runs</item>
/// </list>
/// A <b>collection</b> is a JSON file listing package idents, served from any web server (URL) or stored as a file:
/// <c>["org.pkg", ...]</c> or <c>{ "packages": [ ... ] }</c>. The server mounts everything before players can join;
/// joining clients receive the list with the script manifest and mount the same packages before their scripts start.
/// Packages added while the server runs are mounted on the server and pushed to every connected client.
/// </summary>
public static class ContentPackages
{
	public const int MaxPackages = 128;
	public const int MaxCollections = 16;
	public const int MaxCollectionChars = 64 * 1024;
	public const int CollectionTimeoutSeconds = 15;

	/// <summary>Data-folder base for config collections given as file paths ("collections/x.json" → roleplay/collections/x.json).</summary>
	public const string DataBase = "roleplay";

	static readonly Regex IdentPattern = new( @"^[a-z0-9_\-]{1,64}\.[a-z0-9_\-]{1,64}$" );
	static readonly Regex PackageUrlPattern = new( @"sbox\.game/([a-z0-9_\-]{1,64})/([a-z0-9_\-]{1,64})", RegexOptions.IgnoreCase );

	/// <summary>The current world's required packages (server side; sent to clients).</summary>
	public static IReadOnlyList<string> Required => _required;
	static List<string> _required = new();

	/// <summary>Packages mounted in this process, with their titles.</summary>
	public static IReadOnlyDictionary<string, string> Mounted => _mounted;
	static readonly Dictionary<string, string> _mounted = new();
	static readonly Dictionary<string, string> _primaryAssets = new();

	/// <summary>
	/// Raised after each batch queued through <see cref="Enqueue"/> was mounted: (mounted, failed, shared).
	/// <c>shared</c> = the batch came from the server and is pushed to clients.
	/// </summary>
	public static event Action<List<string>, List<string>, bool> BatchMounted;

	/// <summary>The main asset of a mounted package (e.g. its .vmdl), or null.</summary>
	/// <summary>The mounted package's title, else its ident (for progress text).</summary>
	public static string TitleOrIdent( string ident ) => ident is not null && _mounted.TryGetValue( ident, out var t ) && !string.IsNullOrEmpty( t ) ? t : ident;

	public static string PrimaryAssetOf( string ident ) => ident is not null && _primaryAssets.TryGetValue( ident, out var a ) ? a : null;

	public static bool IsValidIdent( string ident ) => ident is not null && IdentPattern.IsMatch( ident );

	/// <summary>"Org.Name", " org.name ", "https://sbox.game/org/name" → "org.name" (validity is checked separately).</summary>
	public static string Normalize( string raw )
	{
		if ( raw is null ) return null;
		var s = raw.Trim();
		var m = PackageUrlPattern.Match( s );
		if ( m.Success ) s = $"{m.Groups[1].Value}.{m.Groups[2].Value}";
		return s.ToLowerInvariant();
	}

	public static bool IsUrl( string source ) =>
		source is not null && (source.StartsWith( "https://", StringComparison.OrdinalIgnoreCase ) || source.StartsWith( "http://", StringComparison.OrdinalIgnoreCase ));

	/// <summary>Null if <paramref name="source"/> is a usable collection URL or relative .json path, else why not.</summary>
	public static string CheckCollectionSource( string source )
	{
		if ( string.IsNullOrWhiteSpace( source ) ) return "empty collection entry";
		if ( source.Length > 512 ) return "collection entry is longer than 512 characters";
		if ( IsUrl( source ) )
			return Uri.TryCreate( source, UriKind.Absolute, out _ ) ? null : $"'{source}' is not a valid URL";
		if ( source.Contains( ".." ) || source.Contains( '\\' ) || source.StartsWith( "/" ) || source.Contains( ':' ) )
			return $"'{source}' must be a URL or a relative path without '..'";
		if ( !source.EndsWith( ".json", StringComparison.OrdinalIgnoreCase ) ) return $"'{source}' must be a .json file";
		return null;
	}

	// ------------------------------------------------------------ collections

	sealed class CollectionFile
	{
		[System.Text.Json.Serialization.JsonPropertyName( "packages" )]
		public List<string> Packages { get; set; }
	}

	/// <summary>Parses a collection: <c>["a.b", ...]</c> or <c>{ "packages": [...] }</c>. Invalid entries are skipped and counted.</summary>
	public static List<string> ParseCollection( string text, out string error, out int skipped )
	{
		error = null;
		skipped = 0;
		List<string> raw = null;
		var trimmed = (text ?? "").TrimStart();
		if ( trimmed.StartsWith( "[" ) ) Json.TryDeserialize( trimmed, out raw );
		else if ( trimmed.StartsWith( "{" ) && Json.TryDeserialize( trimmed, out CollectionFile file ) ) raw = file?.Packages;

		if ( raw is null ) { error = "not a JSON array of package idents or an object with \"packages\""; return new(); }

		var list = new List<string>();
		foreach ( var r in raw )
		{
			var id = Normalize( r );
			if ( !IsValidIdent( id ) ) { skipped++; continue; }
			if ( !list.Contains( id ) ) list.Add( id );
		}
		return list;
	}

	/// <summary>
	/// Loads one collection. URLs are fetched over HTTP (timeout and size limit); paths are tried against each of
	/// <paramref name="bases"/> (a file system plus folder) in order. Never throws; returns null and an error instead.
	/// </summary>
	public static async Task<(List<string> Packages, string Error)> LoadCollectionAsync( string source, IEnumerable<(BaseFileSystem Fs, string Dir)> bases )
	{
		if ( CheckCollectionSource( source ) is { } bad ) return (null, bad);

		string text;
		string where;
		if ( IsUrl( source ) )
		{
			where = source;
			try
			{
				using var cts = new CancellationTokenSource( TimeSpan.FromSeconds( CollectionTimeoutSeconds ) );
				text = await Http.RequestStringAsync( source, cancellationToken: cts.Token );
			}
			catch ( OperationCanceledException ) { return (null, $"{source}: no answer within {CollectionTimeoutSeconds}s"); }
			catch ( Exception e ) { return (null, $"{source}: {e.Message}"); }
		}
		else
		{
			text = null;
			where = null;
			foreach ( var (fs, dir) in bases ?? Enumerable.Empty<(BaseFileSystem, string)>() )
			{
				if ( fs is null ) continue;
				var path = string.IsNullOrEmpty( dir ) ? source : $"{dir.TrimEnd( '/' )}/{source}";
				if ( !fs.FileExists( path ) ) continue;
				text = fs.ReadAllText( path );
				where = path;
				break;
			}
			if ( text is null ) return (null, $"collection file '{source}' was not found");
		}

		if ( text is null ) return (null, $"{where}: empty answer");
		if ( text.Length > MaxCollectionChars ) return (null, $"{where}: larger than {MaxCollectionChars / 1024} KB");

		var list = ParseCollection( text, out var error, out var skipped );
		if ( error is not null ) return (null, $"{where}: {error}");
		if ( skipped > 0 ) RpLog.Warn( "content", $"{where}: skipped {skipped} invalid entr{(skipped == 1 ? "y" : "ies")} (not org.name idents)" );
		RpLog.Info( "content", $"Collection {where}: {list.Count} package(s)" );
		return (list, null);
	}

	/// <summary>
	/// Server: every package the world needs before it starts. framework.json packages and collections, then each gamemode
	/// of the chain (base first) with its packages and collections (files relative to its folder in <paramref name="luaFs"/>).
	/// Invalid entries and unreachable collections are logged and skipped: content is optional, the server still starts.
	/// </summary>
	public static async Task<List<string>> ResolveAsync( Scripting.GamemodeRegistry registry, string gamemode, BaseFileSystem luaFs, IEnumerable<Scripting.AddonManifest> addons = null )
	{
		var list = new List<string>();
		void Add( IEnumerable<string> src, string from )
		{
			foreach ( var raw in src ?? Enumerable.Empty<string>() )
			{
				var id = Normalize( raw );
				if ( !IsValidIdent( id ) ) { RpLog.Warn( "content", $"Ignoring package '{raw}' from {from}: not an org.name ident" ); continue; }
				if ( !list.Contains( id ) ) list.Add( id );
			}
		}

		var collections = new List<(string Source, string From, (BaseFileSystem, string)[] Bases)>();
		var config = Bootstrap.FrameworkHost.Config;
		Add( config.Packages, "framework.json" );
		foreach ( var c in config.Collections ?? new List<string>() )
			collections.Add( (c, "framework.json", new[] { (FileSystem.Data, DataBase) }) );

		foreach ( var m in registry?.LoadOrder( gamemode ) ?? new List<Scripting.GamemodeManifest>() )
		{
			Add( m.Packages, $"gamemode '{m.Id}'" );
			foreach ( var c in m.Collections ?? new List<string>() )
				collections.Add( (c, $"gamemode '{m.Id}'", new[] { (luaFs, m.Root) }) );
		}

		foreach ( var a in addons ?? Enumerable.Empty<Scripting.AddonManifest>() )
		{
			Add( a.Packages, $"addon '{a.Id}'" );
			foreach ( var c in a.Collections ?? new List<string>() )
				collections.Add( (c, $"addon '{a.Id}'", new[] { (luaFs, a.Root) }) );
		}

		if ( collections.Count > MaxCollections )
		{
			RpLog.Warn( "content", $"{collections.Count} collections configured; only the first {MaxCollections} are loaded" );
			collections = collections.Take( MaxCollections ).ToList();
		}

		foreach ( var (source, from, bases) in collections )
		{
			var (ids, error) = await LoadCollectionAsync( source, bases );
			if ( error is not null ) { RpLog.Warn( "content", $"Collection from {from} skipped: {error}" ); continue; }
			Add( ids, $"collection '{source}'" );
		}

		if ( list.Count > MaxPackages )
		{
			RpLog.Warn( "content", $"{list.Count} packages requested; only the first {MaxPackages} are used" );
			list = list.Take( MaxPackages ).ToList();
		}
		return list;
	}

	// ------------------------------------------------------------ mounting

	/// <summary>Server: sets the world's package list and mounts it. Failures are logged and skipped (content is optional).</summary>
	public static async Task MountRequired( List<string> idents, Action<int, int, string> progress = null )
	{
		_required = new List<string>( idents );
		_queue.Clear();
		await MountAll( idents, progress );
	}

	/// <summary>Mounts packages one by one. Returns the idents that could not be mounted.</summary>
	/// <param name="progress">Called before each package: (index, count, ident) - feeds the loading screen.</param>
	public static async Task<List<string>> MountAll( IEnumerable<string> idents, Action<int, int, string> progress = null )
	{
		var failed = new List<string>();
		var list = idents.ToList();
		for ( int i = 0; i < list.Count; i++ )
		{
			var id = list[i];
			progress?.Invoke( i, list.Count, id );
			if ( _mounted.ContainsKey( id ) ) continue;
			// Let a frame through between packages, so the game (and the loading screen) keeps drawing.
			await GameTask.Yield();
			var started = RealTime.Now;
			try
			{
				var package = await Package.FetchAsync( id, false );
				if ( package is null ) { failed.Add( id ); RpLog.Warn( "content", $"Package '{id}' was not found on sbox.game" ); continue; }
				await package.MountAsync( false );
				_mounted[id] = package.Title;
				if ( !string.IsNullOrEmpty( package.PrimaryAsset ) ) _primaryAssets[id] = package.PrimaryAsset;
				RpLog.Info( "content", $"Mounted '{package.Title}' ({id}, {package.TypeName}) in {RealTime.Now - started:F1}s" );
			}
			catch ( Exception e )
			{
				failed.Add( id );
				RpLog.Warn( "content", $"Could not mount '{id}': {e.Message}" );
			}
		}
		return failed;
	}

	// ------------------------------------------------------------ runtime additions (Lua content library)

	static readonly List<(string Id, bool Share)> _queue = new();
	static Task _pump;
	static int _inflight;

	/// <summary>True while queued packages or collections are still being fetched or mounted.</summary>
	public static bool Busy => _inflight > 0 || _queue.Count > 0 || _pump is { IsCompleted: false };

	/// <summary>
	/// Queues packages to mount. <paramref name="share"/> (server side) adds them to <see cref="Required"/>, so joining
	/// clients get them, and pushes them to connected clients once mounted. Returns how many were new.
	/// </summary>
	public static int Enqueue( IEnumerable<string> idents, bool share )
	{
		int added = 0;
		foreach ( var id in idents )
		{
			if ( !IsValidIdent( id ) ) continue;
			if ( share && _required.Contains( id ) ) continue;
			if ( !share && _mounted.ContainsKey( id ) ) continue;
			if ( _queue.Any( q => q.Id == id && q.Share == share ) ) continue;
			if ( share && _required.Count >= MaxPackages )
			{
				RpLog.Warn( "content", $"Package limit ({MaxPackages}) reached; '{id}' was not added" );
				break;
			}
			if ( share ) _required.Add( id );
			_queue.Add( (id, share) );
			added++;
		}
		if ( added > 0 && _pump is not { IsCompleted: false } ) _pump = Pump();
		return added;
	}

	/// <summary>Fetches a collection in the background, then queues its packages (see <see cref="Enqueue"/>).</summary>
	public static async Task EnqueueCollectionAsync( string source, IEnumerable<(BaseFileSystem Fs, string Dir)> bases, bool share, Action<string> onError )
	{
		_inflight++;
		try
		{
			var (ids, error) = await LoadCollectionAsync( source, bases );
			if ( error is not null ) { RpLog.Warn( "content", $"Collection skipped: {error}" ); onError?.Invoke( error ); return; }
			Enqueue( ids, share );
		}
		finally { _inflight--; }
	}

	static async Task Pump()
	{
		// Never run hooks synchronously inside the Lua call that queued the package (the realm may still be loading).
		await GameTask.DelayRealtimeSeconds( 0.01f );
		while ( _queue.Count > 0 )
		{
			var batch = _queue.ToList();
			_queue.Clear();
			foreach ( var share in new[] { true, false } )
			{
				var ids = batch.Where( q => q.Share == share ).Select( q => q.Id ).Distinct().ToList();
				if ( ids.Count == 0 ) continue;
				var failed = await MountAll( ids );
				if ( share ) _required.RemoveAll( failed.Contains ); // don't make joining clients retry broken packages
				var ok =ids.Where( i => !failed.Contains( i ) && _mounted.ContainsKey( i ) ).ToList();
				try { BatchMounted?.Invoke( ok, failed, share ); }
				catch ( Exception e ) { RpLog.Error( "content", $"ContentMounted handler failed: {e.Message}" ); }
			}
		}
	}

	/// <summary>Waits until queued content is mounted (or <paramref name="timeoutSeconds"/> pass). Used before opening the lobby.</summary>
	public static async Task WhenIdle( double timeoutSeconds )
	{
		var start = RealTime.Now;
		while ( Busy && RealTime.Now - start < timeoutSeconds )
			await GameTask.DelayRealtimeSeconds( 0.05f );
		if ( Busy ) RpLog.Warn( "content", $"Content still loading after {timeoutSeconds:F0}s; opening the server anyway" );
	}

	/// <summary>Client: parses the server's untrusted, comma-separated list.</summary>
	public static List<string> ParseFromServer( string packed )
	{
		return (packed ?? "").Split( ',', StringSplitOptions.RemoveEmptyEntries )
			.Select( s => s.Trim() )
			.Where( IsValidIdent )
			.Distinct()
			.Take( MaxPackages )
			.ToList();
	}
}