Static content package manager for the server. Collects required package idents from framework config, each gamemode in the chain and addon manifests, plus JSON collections loaded over HTTP or from a file, then fetches and mounts them through Package.FetchAsync/MountAsync. Also queues runtime additions from Lua (with a pump task, batch events and a Busy/WhenIdle wait) and parses the server's package list on clients.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using RoleplayFramework.Diagnostics;
namespace RoleplayFramework.Core;
/// <summary>
/// Cloud content a server requires: sbox.game packages (models, materials, sounds, maps, ...; packages published through
/// the Steam Workshop are packages too). Sources, merged and de-duplicated:
/// <list type="bullet">
/// <item>server config <c>"Packages"</c> and <c>"Collections"</c> (framework.json)</item>
/// <item>the gamemode chain's <c>"packages"</c> and <c>"collections"</c> (gamemode.json)</item>
/// <item>Lua: <c>content.AddPackage</c> / <c>content.AddCollection</c>, at load time or while the server runs</item>
/// </list>
/// A <b>collection</b> is a JSON file listing package idents, served from any web server (URL) or stored as a file:
/// <c>["org.pkg", ...]</c> or <c>{ "packages": [ ... ] }</c>. The server mounts everything before players can join;
/// joining clients receive the list with the script manifest and mount the same packages before their scripts start.
/// Packages added while the server runs are mounted on the server and pushed to every connected client.
/// </summary>
public static class ContentPackages
{
public const int MaxPackages = 128;
public const int MaxCollections = 16;
public const int MaxCollectionChars = 64 * 1024;
public const int CollectionTimeoutSeconds = 15;
/// <summary>Data-folder base for config collections given as file paths ("collections/x.json" → roleplay/collections/x.json).</summary>
public const string DataBase = "roleplay";
static readonly Regex IdentPattern = new( @"^[a-z0-9_\-]{1,64}\.[a-z0-9_\-]{1,64}$" );
static readonly Regex PackageUrlPattern = new( @"sbox\.game/([a-z0-9_\-]{1,64})/([a-z0-9_\-]{1,64})", RegexOptions.IgnoreCase );
/// <summary>The current world's required packages (server side; sent to clients).</summary>
public static IReadOnlyList<string> Required => _required;
static List<string> _required = new();
/// <summary>Packages mounted in this process, with their titles.</summary>
public static IReadOnlyDictionary<string, string> Mounted => _mounted;
static readonly Dictionary<string, string> _mounted = new();
static readonly Dictionary<string, string> _primaryAssets = new();
/// <summary>
/// Raised after each batch queued through <see cref="Enqueue"/> was mounted: (mounted, failed, shared).
/// <c>shared</c> = the batch came from the server and is pushed to clients.
/// </summary>
public static event Action<List<string>, List<string>, bool> BatchMounted;
/// <summary>The main asset of a mounted package (e.g. its .vmdl), or null.</summary>
/// <summary>The mounted package's title, else its ident (for progress text).</summary>
public static string TitleOrIdent( string ident ) => ident is not null && _mounted.TryGetValue( ident, out var t ) && !string.IsNullOrEmpty( t ) ? t : ident;
public static string PrimaryAssetOf( string ident ) => ident is not null && _primaryAssets.TryGetValue( ident, out var a ) ? a : null;
public static bool IsValidIdent( string ident ) => ident is not null && IdentPattern.IsMatch( ident );
/// <summary>"Org.Name", " org.name ", "https://sbox.game/org/name" → "org.name" (validity is checked separately).</summary>
public static string Normalize( string raw )
{
if ( raw is null ) return null;
var s = raw.Trim();
var m = PackageUrlPattern.Match( s );
if ( m.Success ) s = $"{m.Groups[1].Value}.{m.Groups[2].Value}";
return s.ToLowerInvariant();
}
public static bool IsUrl( string source ) =>
source is not null && (source.StartsWith( "https://", StringComparison.OrdinalIgnoreCase ) || source.StartsWith( "http://", StringComparison.OrdinalIgnoreCase ));
/// <summary>Null if <paramref name="source"/> is a usable collection URL or relative .json path, else why not.</summary>
public static string CheckCollectionSource( string source )
{
if ( string.IsNullOrWhiteSpace( source ) ) return "empty collection entry";
if ( source.Length > 512 ) return "collection entry is longer than 512 characters";
if ( IsUrl( source ) )
return Uri.TryCreate( source, UriKind.Absolute, out _ ) ? null : $"'{source}' is not a valid URL";
if ( source.Contains( ".." ) || source.Contains( '\\' ) || source.StartsWith( "/" ) || source.Contains( ':' ) )
return $"'{source}' must be a URL or a relative path without '..'";
if ( !source.EndsWith( ".json", StringComparison.OrdinalIgnoreCase ) ) return $"'{source}' must be a .json file";
return null;
}
// ------------------------------------------------------------ collections
sealed class CollectionFile
{
[System.Text.Json.Serialization.JsonPropertyName( "packages" )]
public List<string> Packages { get; set; }
}
/// <summary>Parses a collection: <c>["a.b", ...]</c> or <c>{ "packages": [...] }</c>. Invalid entries are skipped and counted.</summary>
public static List<string> ParseCollection( string text, out string error, out int skipped )
{
error = null;
skipped = 0;
List<string> raw = null;
var trimmed = (text ?? "").TrimStart();
if ( trimmed.StartsWith( "[" ) ) Json.TryDeserialize( trimmed, out raw );
else if ( trimmed.StartsWith( "{" ) && Json.TryDeserialize( trimmed, out CollectionFile file ) ) raw = file?.Packages;
if ( raw is null ) { error = "not a JSON array of package idents or an object with \"packages\""; return new(); }
var list = new List<string>();
foreach ( var r in raw )
{
var id = Normalize( r );
if ( !IsValidIdent( id ) ) { skipped++; continue; }
if ( !list.Contains( id ) ) list.Add( id );
}
return list;
}
/// <summary>
/// Loads one collection. URLs are fetched over HTTP (timeout and size limit); paths are tried against each of
/// <paramref name="bases"/> (a file system plus folder) in order. Never throws; returns null and an error instead.
/// </summary>
public static async Task<(List<string> Packages, string Error)> LoadCollectionAsync( string source, IEnumerable<(BaseFileSystem Fs, string Dir)> bases )
{
if ( CheckCollectionSource( source ) is { } bad ) return (null, bad);
string text;
string where;
if ( IsUrl( source ) )
{
where = source;
try
{
using var cts = new CancellationTokenSource( TimeSpan.FromSeconds( CollectionTimeoutSeconds ) );
text = await Http.RequestStringAsync( source, cancellationToken: cts.Token );
}
catch ( OperationCanceledException ) { return (null, $"{source}: no answer within {CollectionTimeoutSeconds}s"); }
catch ( Exception e ) { return (null, $"{source}: {e.Message}"); }
}
else
{
text = null;
where = null;
foreach ( var (fs, dir) in bases ?? Enumerable.Empty<(BaseFileSystem, string)>() )
{
if ( fs is null ) continue;
var path = string.IsNullOrEmpty( dir ) ? source : $"{dir.TrimEnd( '/' )}/{source}";
if ( !fs.FileExists( path ) ) continue;
text = fs.ReadAllText( path );
where = path;
break;
}
if ( text is null ) return (null, $"collection file '{source}' was not found");
}
if ( text is null ) return (null, $"{where}: empty answer");
if ( text.Length > MaxCollectionChars ) return (null, $"{where}: larger than {MaxCollectionChars / 1024} KB");
var list = ParseCollection( text, out var error, out var skipped );
if ( error is not null ) return (null, $"{where}: {error}");
if ( skipped > 0 ) RpLog.Warn( "content", $"{where}: skipped {skipped} invalid entr{(skipped == 1 ? "y" : "ies")} (not org.name idents)" );
RpLog.Info( "content", $"Collection {where}: {list.Count} package(s)" );
return (list, null);
}
/// <summary>
/// Server: every package the world needs before it starts. framework.json packages and collections, then each gamemode
/// of the chain (base first) with its packages and collections (files relative to its folder in <paramref name="luaFs"/>).
/// Invalid entries and unreachable collections are logged and skipped: content is optional, the server still starts.
/// </summary>
public static async Task<List<string>> ResolveAsync( Scripting.GamemodeRegistry registry, string gamemode, BaseFileSystem luaFs, IEnumerable<Scripting.AddonManifest> addons = null )
{
var list = new List<string>();
void Add( IEnumerable<string> src, string from )
{
foreach ( var raw in src ?? Enumerable.Empty<string>() )
{
var id = Normalize( raw );
if ( !IsValidIdent( id ) ) { RpLog.Warn( "content", $"Ignoring package '{raw}' from {from}: not an org.name ident" ); continue; }
if ( !list.Contains( id ) ) list.Add( id );
}
}
var collections = new List<(string Source, string From, (BaseFileSystem, string)[] Bases)>();
var config = Bootstrap.FrameworkHost.Config;
Add( config.Packages, "framework.json" );
foreach ( var c in config.Collections ?? new List<string>() )
collections.Add( (c, "framework.json", new[] { (FileSystem.Data, DataBase) }) );
foreach ( var m in registry?.LoadOrder( gamemode ) ?? new List<Scripting.GamemodeManifest>() )
{
Add( m.Packages, $"gamemode '{m.Id}'" );
foreach ( var c in m.Collections ?? new List<string>() )
collections.Add( (c, $"gamemode '{m.Id}'", new[] { (luaFs, m.Root) }) );
}
foreach ( var a in addons ?? Enumerable.Empty<Scripting.AddonManifest>() )
{
Add( a.Packages, $"addon '{a.Id}'" );
foreach ( var c in a.Collections ?? new List<string>() )
collections.Add( (c, $"addon '{a.Id}'", new[] { (luaFs, a.Root) }) );
}
if ( collections.Count > MaxCollections )
{
RpLog.Warn( "content", $"{collections.Count} collections configured; only the first {MaxCollections} are loaded" );
collections = collections.Take( MaxCollections ).ToList();
}
foreach ( var (source, from, bases) in collections )
{
var (ids, error) = await LoadCollectionAsync( source, bases );
if ( error is not null ) { RpLog.Warn( "content", $"Collection from {from} skipped: {error}" ); continue; }
Add( ids, $"collection '{source}'" );
}
if ( list.Count > MaxPackages )
{
RpLog.Warn( "content", $"{list.Count} packages requested; only the first {MaxPackages} are used" );
list = list.Take( MaxPackages ).ToList();
}
return list;
}
// ------------------------------------------------------------ mounting
/// <summary>Server: sets the world's package list and mounts it. Failures are logged and skipped (content is optional).</summary>
public static async Task MountRequired( List<string> idents, Action<int, int, string> progress = null )
{
_required = new List<string>( idents );
_queue.Clear();
await MountAll( idents, progress );
}
/// <summary>Mounts packages one by one. Returns the idents that could not be mounted.</summary>
/// <param name="progress">Called before each package: (index, count, ident) - feeds the loading screen.</param>
public static async Task<List<string>> MountAll( IEnumerable<string> idents, Action<int, int, string> progress = null )
{
var failed = new List<string>();
var list = idents.ToList();
for ( int i = 0; i < list.Count; i++ )
{
var id = list[i];
progress?.Invoke( i, list.Count, id );
if ( _mounted.ContainsKey( id ) ) continue;
// Let a frame through between packages, so the game (and the loading screen) keeps drawing.
await GameTask.Yield();
var started = RealTime.Now;
try
{
var package = await Package.FetchAsync( id, false );
if ( package is null ) { failed.Add( id ); RpLog.Warn( "content", $"Package '{id}' was not found on sbox.game" ); continue; }
await package.MountAsync( false );
_mounted[id] = package.Title;
if ( !string.IsNullOrEmpty( package.PrimaryAsset ) ) _primaryAssets[id] = package.PrimaryAsset;
RpLog.Info( "content", $"Mounted '{package.Title}' ({id}, {package.TypeName}) in {RealTime.Now - started:F1}s" );
}
catch ( Exception e )
{
failed.Add( id );
RpLog.Warn( "content", $"Could not mount '{id}': {e.Message}" );
}
}
return failed;
}
// ------------------------------------------------------------ runtime additions (Lua content library)
static readonly List<(string Id, bool Share)> _queue = new();
static Task _pump;
static int _inflight;
/// <summary>True while queued packages or collections are still being fetched or mounted.</summary>
public static bool Busy => _inflight > 0 || _queue.Count > 0 || _pump is { IsCompleted: false };
/// <summary>
/// Queues packages to mount. <paramref name="share"/> (server side) adds them to <see cref="Required"/>, so joining
/// clients get them, and pushes them to connected clients once mounted. Returns how many were new.
/// </summary>
public static int Enqueue( IEnumerable<string> idents, bool share )
{
int added = 0;
foreach ( var id in idents )
{
if ( !IsValidIdent( id ) ) continue;
if ( share && _required.Contains( id ) ) continue;
if ( !share && _mounted.ContainsKey( id ) ) continue;
if ( _queue.Any( q => q.Id == id && q.Share == share ) ) continue;
if ( share && _required.Count >= MaxPackages )
{
RpLog.Warn( "content", $"Package limit ({MaxPackages}) reached; '{id}' was not added" );
break;
}
if ( share ) _required.Add( id );
_queue.Add( (id, share) );
added++;
}
if ( added > 0 && _pump is not { IsCompleted: false } ) _pump = Pump();
return added;
}
/// <summary>Fetches a collection in the background, then queues its packages (see <see cref="Enqueue"/>).</summary>
public static async Task EnqueueCollectionAsync( string source, IEnumerable<(BaseFileSystem Fs, string Dir)> bases, bool share, Action<string> onError )
{
_inflight++;
try
{
var (ids, error) = await LoadCollectionAsync( source, bases );
if ( error is not null ) { RpLog.Warn( "content", $"Collection skipped: {error}" ); onError?.Invoke( error ); return; }
Enqueue( ids, share );
}
finally { _inflight--; }
}
static async Task Pump()
{
// Never run hooks synchronously inside the Lua call that queued the package (the realm may still be loading).
await GameTask.DelayRealtimeSeconds( 0.01f );
while ( _queue.Count > 0 )
{
var batch = _queue.ToList();
_queue.Clear();
foreach ( var share in new[] { true, false } )
{
var ids = batch.Where( q => q.Share == share ).Select( q => q.Id ).Distinct().ToList();
if ( ids.Count == 0 ) continue;
var failed = await MountAll( ids );
if ( share ) _required.RemoveAll( failed.Contains ); // don't make joining clients retry broken packages
var ok =ids.Where( i => !failed.Contains( i ) && _mounted.ContainsKey( i ) ).ToList();
try { BatchMounted?.Invoke( ok, failed, share ); }
catch ( Exception e ) { RpLog.Error( "content", $"ContentMounted handler failed: {e.Message}" ); }
}
}
}
/// <summary>Waits until queued content is mounted (or <paramref name="timeoutSeconds"/> pass). Used before opening the lobby.</summary>
public static async Task WhenIdle( double timeoutSeconds )
{
var start = RealTime.Now;
while ( Busy && RealTime.Now - start < timeoutSeconds )
await GameTask.DelayRealtimeSeconds( 0.05f );
if ( Busy ) RpLog.Warn( "content", $"Content still loading after {timeoutSeconds:F0}s; opening the server anyway" );
}
/// <summary>Client: parses the server's untrusted, comma-separated list.</summary>
public static List<string> ParseFromServer( string packed )
{
return (packed ?? "").Split( ',', StringSplitOptions.RemoveEmptyEntries )
.Select( s => s.Trim() )
.Where( IsValidIdent )
.Distinct()
.Take( MaxPackages )
.ToList();
}
}