Gliner/Packaging/Sbgli1Manifest.cs
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.Text;

namespace GlinerPoc.Packaging;

/// <summary>
/// C# parser/validator for the SBGLI1 binary manifest produced by the
/// deterministic Python exporter (format version 1). Mirrors the on-disk
/// layout exactly; every failure throws with an actionable message.
///
/// Layout (little-endian):
///   header 128 B: magic(8) ver flags headerBytes tensorCount chunkCount
///                 shardCount dtype reserved ckptSha(32) rcSha(32) zero(24)
///   tensor records, chunk records, shard records, recordsSha(32)
/// </summary>
public sealed class Sbgli1Manifest
{
	public const string Magic = "SBGLI1";
	public const int FormatVersion = 1;
	public const int HeaderBytes = 128;
	public const int StorageDtypeF32 = 1;

	public sealed class TensorRecord
	{
		public string Name;
		public int[] Shape;
		public int Rank;
		public long ElementCount;
		public long ByteCount;
		public int FirstChunkIndex;
		public int ChunkCount;
	}

	public sealed class ChunkRecord
	{
		public int TensorIndex;
		public int ChunkIndex;
		public long ElemOffset;
		public long ElemCount;
		public int RowStart;
		public int RowCount;
		public int ShardIndex;
		public long ByteOffsetInShard;
		public string Sha256;
	}

	public sealed class ShardRecord
	{
		public int Index;
		public long ByteCount;
		public string Sha256;
	}

	public int FormatVersionField;
	public int TensorCount;
	public int ChunkCount;
	public int ShardCount;
	public string CheckpointSha256;
	public string RuntimeConfigSha256;

	public List<TensorRecord> Tensors = new();
	public List<ChunkRecord> Chunks = new();
	public List<ShardRecord> Shards = new();

	private readonly Dictionary<string, int> _tensorIndexByName = new( StringComparer.Ordinal );

	public static Sbgli1Manifest Parse( byte[] data )
	{
		var m = new Sbgli1Manifest();
		int pos = 0;

		string magic = Encoding.ASCII.GetString( data, 0, 8 ).TrimEnd( '\0' );
		if ( magic != Magic )
			throw new InvalidOperationException( $"[GLI:ERROR] SBGLI1 magic expected '{Magic}', found '{magic}'." );
		pos = 8;

		m.FormatVersionField = (int)ReadU32( data, ref pos );
		int flags = (int)ReadU32( data, ref pos );
		int headerBytes = (int)ReadU32( data, ref pos );
		if ( m.FormatVersionField != FormatVersion )
			throw new InvalidOperationException( $"[GLI:ERROR] SBGLI1 format version expected {FormatVersion}, found {m.FormatVersionField}." );
		if ( (flags & 1) == 0 )
			throw new InvalidOperationException( "[GLI:ERROR] SBGLI1 big-endian data is unsupported." );
		if ( headerBytes != HeaderBytes )
			throw new InvalidOperationException( $"[GLI:ERROR] SBGLI1 header size expected {HeaderBytes}, found {headerBytes}." );

		m.TensorCount = (int)ReadU32( data, ref pos );
		m.ChunkCount = (int)ReadU32( data, ref pos );
		m.ShardCount = (int)ReadU32( data, ref pos );
		int dtype = (int)ReadU32( data, ref pos );
		int reserved = (int)ReadU32( data, ref pos );
		if ( dtype != StorageDtypeF32 )
			throw new InvalidOperationException( $"[GLI:ERROR] SBGLI1 storage dtype expected {StorageDtypeF32} (FP32), found {dtype}." );
		if ( reserved != 0 )
			throw new InvalidOperationException( "[GLI:ERROR] SBGLI1 reserved header field must be zero." );

		m.CheckpointSha256 = Hex( data, pos, 32 );
		pos += 32;
		m.RuntimeConfigSha256 = Hex( data, pos, 32 );
		pos += 32;
		for ( int i = 0; i < 24; i++ )
		{
			if ( data[pos + i] != 0 )
				throw new InvalidOperationException( "[GLI:ERROR] SBGLI1 reserved header tail must be zero." );
		}
		pos += 24;
		if ( pos != HeaderBytes )
			throw new InvalidOperationException( "[GLI:ERROR] SBGLI1 header size mismatch." );

		int recordsStart = pos;

		for ( int i = 0; i < m.TensorCount; i++ )
		{
			var t = new TensorRecord();
			int nameLen = data[pos];
			pos += 1;
			t.Name = Encoding.UTF8.GetString( data, pos, nameLen );
			pos += nameLen;
			t.Rank = data[pos];
			pos += 1;
			t.Shape = new int[t.Rank];
			for ( int d = 0; d < t.Rank; d++ )
			{
				t.Shape[d] = (int)ReadU32( data, ref pos );
			}
			t.ElementCount = (long)ReadU64( data, ref pos );
			t.ByteCount = (long)ReadU64( data, ref pos );
			t.FirstChunkIndex = (int)ReadU32( data, ref pos );
			t.ChunkCount = (int)ReadU32( data, ref pos );

			if ( t.Rank <= 0 || t.Rank > 8 )
				throw new InvalidOperationException( $"[GLI:ERROR] Tensor '{t.Name}' has invalid rank {t.Rank}." );
			if ( t.ElementCount <= 0 || t.ByteCount != t.ElementCount * 4 )
				throw new InvalidOperationException( $"[GLI:ERROR] Tensor '{t.Name}' element/byte count mismatch ({t.ElementCount} / {t.ByteCount})." );
			if ( m._tensorIndexByName.ContainsKey( t.Name ) )
				throw new InvalidOperationException( $"[GLI:ERROR] Duplicate tensor record '{t.Name}'." );
			m._tensorIndexByName[t.Name] = i;
			m.Tensors.Add( t );
		}

		for ( int i = 0; i < m.ChunkCount; i++ )
		{
			var c = new ChunkRecord
			{
				TensorIndex = (int)ReadU32( data, ref pos ),
				ChunkIndex = (int)ReadU32( data, ref pos ),
				ElemOffset = (long)ReadU64( data, ref pos ),
				ElemCount = (long)ReadU64( data, ref pos ),
				RowStart = (int)ReadU32( data, ref pos ),
				RowCount = (int)ReadU32( data, ref pos ),
				ShardIndex = (int)ReadU32( data, ref pos ),
				ByteOffsetInShard = (long)ReadU64( data, ref pos ),
				Sha256 = Hex( data, pos, 32 ),
			};
			pos += 32;
			if ( c.TensorIndex < 0 || c.TensorIndex >= m.TensorCount )
				throw new InvalidOperationException( $"[GLI:ERROR] Chunk {i} references tensor {c.TensorIndex} out of bounds." );
			if ( c.ShardIndex < 0 || c.ShardIndex >= m.ShardCount )
				throw new InvalidOperationException( $"[GLI:ERROR] Chunk {i} references shard {c.ShardIndex} out of bounds." );
			var owner = m.Tensors[c.TensorIndex];
			if ( c.ElemOffset < 0 || c.ElemOffset + c.ElemCount > owner.ElementCount )
				throw new InvalidOperationException( $"[GLI:ERROR] Chunk {i} element range outside tensor '{owner.Name}'." );
			m.Chunks.Add( c );
		}

		for ( int i = 0; i < m.ShardCount; i++ )
		{
			var s = new ShardRecord
			{
				Index = (int)ReadU32( data, ref pos ),
				ByteCount = (long)ReadU64( data, ref pos ),
				Sha256 = Hex( data, pos, 32 ),
			};
			pos += 32;
			m.Shards.Add( s );
		}

		int recordsEnd = pos;
		string recordsSha = Convert.ToHexString( SHA256.HashData(
			data[recordsStart..recordsEnd] ) ).ToLowerInvariant();
		string storedRecordsSha = Hex( data, pos, 32 );
		pos += 32;
		if ( recordsSha != storedRecordsSha )
			throw new InvalidOperationException(
				$"[GLI:ERROR] SBGLI1 records region SHA-256 mismatch (expected {storedRecordsSha}, computed {recordsSha})." );
		if ( pos != data.Length )
			throw new InvalidOperationException( $"[GLI:ERROR] SBGLI1 manifest has {data.Length - pos} trailing bytes." );
		return m;
	}

	public TensorRecord GetRequiredTensor( string name )
	{
		if ( !_tensorIndexByName.TryGetValue( name, out int index ) )
			throw new InvalidOperationException( $"[GLI:ERROR] Required tensor '{name}' is missing from the SBGLI1 manifest." );
		return Tensors[index];
	}

	/// <summary>
	/// Copies the exact bytes of one tensor from the provided shard payloads.
	/// shardData must be indexed by shard record index. Every chunk is
	/// SHA-256-verified before copying.
	/// </summary>
	public byte[] ReadTensorBytes( string name, IReadOnlyList<byte[]> shardData )
	{
		var t = GetRequiredTensor( name );
		var result = new byte[t.ByteCount];
		int written = 0;
		for ( int ci = t.FirstChunkIndex; ci < t.FirstChunkIndex + t.ChunkCount; ci++ )
		{
			var c = Chunks[ci];
			byte[] shard = shardData[c.ShardIndex];
			long off = c.ByteOffsetInShard;
			int len = (int)(c.ElemCount * 4);
			if ( off < 0 || off + len > shard.Length )
				throw new InvalidOperationException(
					$"[GLI:ERROR] Chunk {ci} of '{name}' reads [{off}, {off + len}) outside shard {c.ShardIndex} ({shard.Length} B)." );
			byte[] payload = shard[(int)off..(int)(off + len)];
			string sha = Convert.ToHexString( SHA256.HashData( payload ) ).ToLowerInvariant();
			if ( sha != c.Sha256 )
				throw new InvalidOperationException( $"[GLI:ERROR] Chunk {ci} of '{name}' SHA-256 mismatch." );
			Buffer.BlockCopy( payload, 0, result, written, len );
			written += len;
		}
		if ( written != t.ByteCount )
			throw new InvalidOperationException( $"[GLI:ERROR] Tensor '{name}' reconstructed {written} of {t.ByteCount} bytes." );
		return result;
	}

	private static ulong ReadU32( byte[] data, ref int pos )
	{
		ulong v = BitConverter.ToUInt32( data, pos );
		pos += 4;
		return v;
	}

	private static ulong ReadU64( byte[] data, ref int pos )
	{
		ulong v = BitConverter.ToUInt64( data, pos );
		pos += 8;
		return v;
	}

	private static string Hex( byte[] data, int offset, int count )
	{
		return Convert.ToHexString( data, offset, count ).ToLowerInvariant();
	}
}