swb_base/GunAudioPacks.cs

Gun audio pack manager and loader. It reads per-weapon .gunpack files index-first, loads pack bodies in background, decodes individual recordings on demand or prepares sets ahead of time, and exposes console commands and a GameObjectSystem to start loading during scene load/initialization.

File Access
using Sandbox;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading.Tasks;

namespace SWB.Base;

/// <summary>
/// GUN AUDIO PACKS — package trim, step 4 (2026-10-03, Docs/STEP4_GUN_PACKS.md).
///
/// Every gun recording was its own file (a `.vsnd_c`; 7,384 of them are played only by converted guns), and the package's
/// file list has to stay under 8 MiB. `Tools/gun_packs.py` puts them into a few PACK files, one per weapon pack
/// (`sounds/weapons/_packs/*.gunpack`, the format in GunPackCodec), and a converted cue names its recordings in
/// `GunCue.Packed`. This reads the packs and makes engine sounds out of them, one gun at a time.
///
/// The user's two rules (2026-10-03): fewer files, and nothing in game may get noticeably slower; memory is not a
/// concern and a loading screen is fine. So:
/// - **The packs are read while the game loads** (`GunAudioPackLoader`): the index first, on the main thread, in
///   milliseconds; then the packs themselves, whole, into memory, in the background. In game nothing reads a pack.
/// - **A gun's recordings are cut when the gun arrives** (`Prepare`): unpacked on a worker thread, handed to the engine
///   (`SoundFile.FromPcm`) on the main thread. Measured on the user's PC before any of this was built: 2 ms for a
///   typical gun, 15 ms for the biggest, all of it on the worker but the hand-off. The trial's log (2026-10-03 17:13):
///   1–2.6 ms on the main thread per gun.
///   - A weapon cuts its own in `OnStart`.
///   - The mystery box's gun is cut the moment it is rolled (`NZNet.BoxRolled` reaches every machine), during the spin.
/// - **Anything wanted before it was cut is cut on the spot** (`Get`): that one recording, on the main thread, and logged.
///   That is the draw sound of a gun that has just arrived, or another player's first shot.
///
/// ⛔ NEVER A WHOLE PACK ON THE MAIN THREAD. A recording wanted while the packs are still being read (a player who joined a
/// moment ago, hearing a shot) is read from the disk ALONE: a few hundred KB, not 290 MB.
/// ⛔ CUT ONCE, KEPT FOR THE SESSION. A recording's sound is cached by name; 200 different guns are about 400 MB.
/// ⚠️ ONLY A GAME IN PLAY READS THE PACKS. A GameObjectSystem also runs in the editor's own scenes (INSTRUCTIONS.md).
/// </summary>
public static class GunAudioPacks
{
	/// <summary>Where the packs live: every `*.gunpack` here is read. A Resources glob in the `.sbproj` ships them.</summary>
	public const string Folder = "sounds/weapons/_packs";

	struct Entry
	{
		public int Pack;
		public int Offset;
		public int Length;
	}

	// ⚠️ NAMES NO OLDER STATIC HAS: hotload carries statics forward by name (INSTRUCTIONS.md). These were renamed when
	// the loading became index-first, so a session hotloaded across the change never mixes the two.
	static Dictionary<string, Entry> _packIndex;
	static byte[][] _packBodies;
	static List<string> _packFiles;
	static Task _packBodyLoad;
	static readonly Dictionary<string, SoundFile> _gunPackSounds = new( StringComparer.OrdinalIgnoreCase );
	static readonly HashSet<string> _gunPackPending = new( StringComparer.OrdinalIgnoreCase );
	static readonly HashSet<string> _gunPackMissing = new( StringComparer.OrdinalIgnoreCase );
	static long _gunPackPcmBytes;
	static int _gunPackCutOnTheSpot;

	static double Ms( long since ) => (Stopwatch.GetTimestamp() - since) * 1000.0 / Stopwatch.Frequency;

	/// <summary>Have the packs been read, all of them, into memory.</summary>
	public static bool IsLoaded => _packBodies is not null && _packBodies.All( b => b is not null );

	/// <summary>
	/// Read the packs: the index now, the packs themselves in the background. Called while the game loads; nothing once
	/// they are read.
	/// </summary>
	public static Task LoadAsync()
	{
		EnsureIndex();
		return _packBodyLoad ??= LoadBodiesAsync();
	}

	static List<string> PackFiles()
	{
		var fs = FileSystem.Mounted;
		if ( fs is null || !fs.DirectoryExists( Folder ) ) return new();
		return fs.FindFile( Folder, "*.gunpack" )
			.OrderBy( f => f, StringComparer.OrdinalIgnoreCase )
			.Select( f => $"{Folder}/{f}" )
			.ToList();
	}

	static byte[] ReadExactly( Stream s, int n )
	{
		var b = new byte[n];
		int got = 0;
		while ( got < n )
		{
			int r = s.Read( b, got, n - got );
			if ( r <= 0 ) throw new EndOfStreamException();
			got += r;
		}
		return b;
	}

	/// <summary>
	/// Which recording is where: each pack's header and index, read from the start of its file. ⚠️ ONLY THE INDEX, so
	/// it costs milliseconds whenever it is first needed; the recordings themselves come with `LoadBodiesAsync`.
	/// </summary>
	static void EnsureIndex()
	{
		if ( _packIndex is not null ) return;

		var t0 = Stopwatch.GetTimestamp();
		var files = PackFiles();
		var index = new Dictionary<string, Entry>( StringComparer.OrdinalIgnoreCase );
		for ( int i = 0; i < files.Count; i++ )
		{
			try
			{
				using var s = FileSystem.Mounted.OpenRead( files[i] );
				var head = ReadExactly( s, 12 );
				if ( head[0] != 'N' || head[1] != 'Z' || head[2] != 'G' || head[3] != 'P'
					|| GunPackCodec.I32( head, 4 ) != GunPackCodec.Version )
				{
					Log.Warning( $"[gun-packs] {files[i]} is not a version-{GunPackCodec.Version} pack: the guns that use it are silent" );
					continue;
				}

				int length = GunPackCodec.I32( head, 8 );
				using var doc = JsonDocument.Parse( ReadExactly( s, length ) );
				int data = 12 + length;
				foreach ( var e in doc.RootElement.GetProperty( "entries" ).EnumerateArray() )
				{
					var name = e[0].GetString();
					// ⚠️ THE FIRST PACK WINS. The tool puts a name in one pack only, so a second copy is a tool error.
					if ( !string.IsNullOrEmpty( name ) && !index.ContainsKey( name ) )
						index[name] = new Entry { Pack = i, Offset = data + e[1].GetInt32(), Length = e[2].GetInt32() };
				}
			}
			catch ( Exception ex )
			{
				Log.Warning( $"[gun-packs] {files[i]}: its index would not read ({ex.Message})" );
			}
		}

		_packFiles = files;
		_packBodies = new byte[files.Count][];
		_packIndex = index;
		Log.Info( $"[gun-packs] index: {files.Count} pack(s), {index.Count} recordings, read in {Ms( t0 ):0.0} ms" );
	}

	static async Task LoadBodiesAsync()
	{
		try
		{
			var t0 = Stopwatch.GetTimestamp();
			var files = _packFiles;
			var bodies = _packBodies;
			long total = 0;
			for ( int i = 0; i < files.Count; i++ )
			{
				if ( bodies[i] is not null ) continue;
				var b = await FileSystem.Mounted.ReadAllBytesAsync( files[i] );
				await GameTask.MainThread();
				bodies[i] = b;
				total += b?.Length ?? 0;
			}
			Log.Info( $"[gun-packs] read while loading: {files.Count} pack(s), {total / 1048576.0:0.0} MB in {Ms( t0 ):0} ms" );
			WarmUp();
		}
		catch ( Exception ex )
		{
			// ⚠️ NOT A FAULTED TASK FOR EVER: the next call tries again, and meanwhile a recording is read from the disk alone
			Log.Warning( $"[gun-packs] reading the packs failed ({ex.Message}); a recording is read from the disk when it is wanted" );
			_packBodyLoad = null;
		}
	}

	/// <summary>
	/// ⚠️ THE FIRST CUT OF A SESSION PAYS FOR COMPILING THE DECODER (measured: 12 ms for a typical gun, against 2 once it is
	/// compiled). One small recording unpacked here, while the game loads, pays it here instead.
	/// </summary>
	static void WarmUp()
	{
		var loaded = _packIndex.Values.Where( x => _packBodies[x.Pack] is not null ).ToList();
		if ( loaded.Count == 0 ) return;
		var e = loaded.OrderBy( x => x.Length ).First();
		var t0 = Stopwatch.GetTimestamp();
		GunPackCodec.Decode( _packBodies[e.Pack], e.Offset, e.Length, out _, out _ );
		Log.Info( $"[gun-packs] decoder ready in {Ms( t0 ):0.0} ms" );
	}

	/// <summary>
	/// A recording's bytes: its pack in memory, or ⛔ WHILE THE PACKS ARE STILL BEING READ, this recording alone, read from
	/// the disk now (a few hundred KB, about a millisecond). Never a whole pack.
	/// </summary>
	static byte[] Bytes( Entry e, out int offset, out bool fromDisk )
	{
		var body = _packBodies[e.Pack];
		if ( body is not null )
		{
			offset = e.Offset;
			fromDisk = false;
			return body;
		}

		using var s = FileSystem.Mounted.OpenRead( _packFiles[e.Pack] );
		if ( s.CanSeek )
		{
			s.Seek( e.Offset, SeekOrigin.Begin );
		}
		else
		{
			var skip = new byte[81920];
			long left = e.Offset;
			while ( left > 0 )
			{
				int r = s.Read( skip, 0, (int)Math.Min( skip.Length, left ) );
				if ( r <= 0 ) throw new EndOfStreamException();
				left -= r;
			}
		}
		offset = 0;
		fromDisk = true;
		return ReadExactly( s, e.Length );
	}

	/// <summary>
	/// A packed recording as an engine sound: already cut (the usual case), or cut now, on this thread, and logged.
	/// Null when no pack has it (said once).
	/// </summary>
	public static SoundFile Get( string name )
	{
		if ( string.IsNullOrEmpty( name ) ) return null;
		if ( _gunPackSounds.TryGetValue( name, out var s ) ) return s;
		EnsureIndex();

		if ( !_packIndex.TryGetValue( name, out var e ) )
		{
			if ( _gunPackMissing.Add( name ) ) Log.Warning( $"[gun-packs] no pack has '{name}': that sound is silent" );
			return null;
		}

		var t0 = Stopwatch.GetTimestamp();
		byte[] pcm;
		int ch, rate;
		bool fromDisk;
		try
		{
			var src = Bytes( e, out int offset, out fromDisk );
			pcm = GunPackCodec.Decode( src, offset, e.Length, out ch, out rate );
		}
		catch ( Exception ex )
		{
			if ( _gunPackMissing.Add( name ) ) Log.Warning( $"[gun-packs] '{name}' would not read ({ex.Message}): that sound is silent" );
			return null;
		}

		s = Make( name, pcm, ch, rate );
		_gunPackCutOnTheSpot++;
		Log.Info( $"[gun-packs] cut '{name}' on the spot, on the main thread, in {Ms( t0 ):0.00} ms ("
			+ (fromDisk ? "read from the disk alone: its pack was still being read)" : "it had not been cut ahead)") );
		return s;
	}

	static SoundFile Make( string name, byte[] pcm, int channels, int rate )
	{
		if ( pcm is null || pcm.Length == 0 )
		{
			if ( _gunPackMissing.Add( name ) ) Log.Warning( $"[gun-packs] '{name}' would not unpack: that sound is silent" );
			return null;
		}

		var s = SoundFile.FromPcm( "gunpack:" + name, pcm.AsSpan(),
			new SoundFile.PcmOptions { Bits = 16, Channels = channels, Rate = (uint)rate } );
		if ( s is null ) return null;

		_gunPackSounds[name] = s;
		_gunPackPcmBytes += pcm.Length;
		return s;
	}

	/// <summary>
	/// Cut these recordings ahead of need: unpacked on a worker thread, handed to the engine on the main thread. Fire and
	/// forget. One already cut, or being cut, is skipped; one whose pack could not be read is left to `Get`.
	/// </summary>
	public static async Task Prepare( IEnumerable<string> names, string why )
	{
		List<string> want = null;
		try
		{
			want = names?.Where( n => !string.IsNullOrEmpty( n ) ).Distinct( StringComparer.OrdinalIgnoreCase ).ToList();
			if ( want is null || want.Count == 0 ) return;

			await LoadAsync();
			if ( _packIndex is null || _packBodies is null ) return;

			want = want.Where( n => !_gunPackSounds.ContainsKey( n ) && !_gunPackPending.Contains( n ) ).ToList();
			if ( want.Count == 0 ) return;
			foreach ( var n in want ) _gunPackPending.Add( n );

			var bodies = _packBodies;
			var jobs = new List<(string name, Entry e)>();
			foreach ( var n in want )
				if ( _packIndex.TryGetValue( n, out var e ) && bodies[e.Pack] is not null ) jobs.Add( (n, e) );
			if ( jobs.Count == 0 ) return;

			var t0 = Stopwatch.GetTimestamp();
			var cut = await GameTask.RunInThreadAsync( () =>
			{
				// ⚠️ PURE WORK ON PRIVATE DATA: a pack is never written after it is read, and nothing here touches the engine
				var list = new List<(string name, byte[] pcm, int ch, int rate)>( jobs.Count );
				foreach ( var j in jobs )
				{
					byte[] pcm = null;
					int ch = 0, rate = 0;
					try { pcm = GunPackCodec.Decode( bodies[j.e.Pack], j.e.Offset, j.e.Length, out ch, out rate ); }
					catch ( Exception ) { }
					list.Add( (j.name, pcm, ch, rate) );
				}
				return list;
			} );
			double worker = Ms( t0 );

			await GameTask.MainThread();
			var t1 = Stopwatch.GetTimestamp();
			int made = 0;
			foreach ( var c in cut )
				if ( !_gunPackSounds.ContainsKey( c.name ) && Make( c.name, c.pcm, c.ch, c.rate ) is not null ) made++;

			if ( made > 0 )
				Log.Info( $"[gun-packs] {why}: {made} recording(s) cut ahead, {worker:0.0} ms until the worker thread was done, "
					+ $"{Ms( t1 ):0.00} ms on the main thread" );
		}
		catch ( Exception ex )
		{
			Log.Warning( $"[gun-packs] {why}: cutting ahead failed ({ex.Message}); they are cut when first played" );
		}
		finally
		{
			if ( want is not null )
				foreach ( var n in want ) _gunPackPending.Remove( n );
		}
	}

	/// <summary>A gun's packed recordings, cut ahead, from its prefab: the mystery box's roll, during the spin.</summary>
	public static void PrepareGun( string prefabPath, string why )
	{
		if ( string.IsNullOrEmpty( prefabPath ) ) return;
		try
		{
			var root = ResourceLibrary.Get<PrefabFile>( prefabPath )?.RootObject;
			if ( root is null ) return;
			var names = new List<string>();
			CollectPacked( root, names );
			if ( names.Count > 0 ) _ = Prepare( names, why );
		}
		catch ( Exception ex )
		{
			Log.Warning( $"[gun-packs] {prefabPath}: could not list its packed sounds ({ex.Message})" );
		}
	}

	static void CollectPacked( JsonNode node, List<string> into )
	{
		if ( node is JsonObject o )
		{
			foreach ( var kv in o )
			{
				if ( kv.Key == "Packed" && kv.Value is JsonArray a )
				{
					foreach ( var x in a )
						if ( x is JsonValue v && v.TryGetValue<string>( out var s ) && !string.IsNullOrEmpty( s ) ) into.Add( s );
				}
				else if ( kv.Value is not null )
				{
					CollectPacked( kv.Value, into );
				}
			}
		}
		else if ( node is JsonArray arr )
		{
			foreach ( var x in arr )
				if ( x is not null ) CollectPacked( x, into );
		}
	}

	/// <summary>`nz_gunpacks`: the packs, and how many recordings are cut.</summary>
	[ConCmd( "nz_gunpacks" )]
	public static void Status()
	{
		if ( _packIndex is null )
		{
			Log.Info( "[gun-packs] not read yet" );
			return;
		}

		Log.Info( $"[gun-packs] {_packFiles.Count} pack(s), {_packIndex.Count} recordings; {_gunPackSounds.Count} cut "
			+ $"({_gunPackPcmBytes / 1048576.0:0.0} MB), {_gunPackCutOnTheSpot} of them on the spot; {_gunPackPending.Count} being cut" );
		for ( int i = 0; i < _packFiles.Count; i++ )
			Log.Info( $"   {_packFiles[i]}  "
				+ (_packBodies[i] is { } b ? $"{b.Length / 1048576.0:0.0} MB" : "still being read") );
	}

	/// <summary>`nz_gunpacks_bench`: unpack every packed recording now, on the main thread, and time it. Nothing is kept.</summary>
	[ConCmd( "nz_gunpacks_bench" )]
	public static void Bench()
	{
		EnsureIndex();
		var t0 = Stopwatch.GetTimestamp();
		long bytes = 0;
		double audio = 0;
		int n = 0;
		foreach ( var e in _packIndex.Values )
		{
			var src = Bytes( e, out int offset, out _ );
			var pcm = GunPackCodec.Decode( src, offset, e.Length, out int ch, out int rate );
			if ( pcm is null ) continue;
			bytes += pcm.Length;
			audio += pcm.Length / 2.0 / ch / rate;
			n++;
		}
		double ms = Ms( t0 );
		Log.Info( $"[gun-packs] bench: {n} recordings, {audio:0.0} s of audio unpacked in {ms:0.0} ms on the main thread, "
			+ $"{ms / Math.Max( audio, 0.001 ):0.00} ms per second of audio ({bytes / 1048576.0:0.0} MB of PCM, not kept)" );
	}
}

/// <summary>
/// Reads the gun audio packs while a game loads (GunAudioPacks).
/// ⚠️ TWO ENTRY POINTS, ONE LOAD: `ISceneLoadingEvents.OnLoad`, and `OnClientInitialize` on every machine in case the
/// first never ran. Both do nothing once the packs are read. (The trial's log showed the load starting in `OnLoad` and
/// finishing after `OnHostInitialize`, so the engine does not always wait for it; nothing depends on it waiting.)
/// ⚠️ NOT IN THE EDITOR'S OWN SCENES: only while a game is playing.
/// </summary>
public sealed class GunAudioPackLoader : GameObjectSystem<GunAudioPackLoader>, ISceneLoadingEvents, ISceneStartup
{
	public GunAudioPackLoader( Scene scene ) : base( scene ) { }

	Task ISceneLoadingEvents.OnLoad( Scene scene, SceneLoadOptions options )
		=> Game.IsPlaying ? GunAudioPacks.LoadAsync() : Task.CompletedTask;

	void ISceneStartup.OnHostPreInitialize( SceneFile scene ) { }

	void ISceneStartup.OnHostInitialize() { }

	void ISceneStartup.OnClientInitialize() { _ = GunAudioPacks.LoadAsync(); }
}