Editor/TextToAnimation/Inference/Runtime/RemoteZip.cs

Utilities for reading ZIP files via ranged access. Defines IRangeSource for random-access reads, FileRangeSource for local files, HttpRangeSource for HTTP range requests with retries and stall timeout, RemoteZip for parsing central directory and extracting STORED entries, and a CRC-32 implementation.

NetworkingFile AccessHttp Calls
#nullable enable
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace TextToAnimation.Editor.Inference.Runtime;

/// <summary>Random access to a file's bytes (a local file, or HTTP range requests).</summary>
public interface IRangeSource
{
	long Length { get; }
	Task<byte[]> ReadAsync( long offset, int count, CancellationToken token );
	/// <summary>Copies [offset, offset+count) into a stream, reporting bytes copied.</summary>
	Task CopyAsync( long offset, long count, Stream destination, Action<long>? progress, CancellationToken token );
}

public sealed class FileRangeSource : IRangeSource
{
	readonly string _path;
	public FileRangeSource( string path ) { _path = path; Length = new FileInfo( path ).Length; }
	public long Length { get; }

	public async Task<byte[]> ReadAsync( long offset, int count, CancellationToken token )
	{
		await using var fs = new FileStream( _path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, useAsync: true );
		fs.Seek( offset, SeekOrigin.Begin );
		var buffer = new byte[count];
		await fs.ReadExactlyAsync( buffer, token );
		return buffer;
	}

	public async Task CopyAsync( long offset, long count, Stream destination, Action<long>? progress, CancellationToken token )
	{
		await using var fs = new FileStream( _path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, useAsync: true );
		fs.Seek( offset, SeekOrigin.Begin );
		var buffer = new byte[1 << 20];
		long done = 0;
		while ( done < count )
		{
			var n = await fs.ReadAsync( buffer.AsMemory( 0, (int)Math.Min( buffer.Length, count - done ) ), token );
			if ( n <= 0 ) throw new EndOfStreamException();
			await destination.WriteAsync( buffer.AsMemory( 0, n ), token );
			done += n;
			progress?.Invoke( done );
		}
	}
}

/// <summary>HTTP range reads with a stall timeout and retries (redirects to the CDN are followed).</summary>
public sealed class HttpRangeSource : IRangeSource
{
	static readonly HttpClient Http = CreateClient();
	readonly string _url;
	static readonly TimeSpan StallTimeout = TimeSpan.FromSeconds( 45 );

	static HttpClient CreateClient()
	{
		var client = new HttpClient( new HttpClientHandler { AllowAutoRedirect = true } ) { Timeout = Timeout.InfiniteTimeSpan };
		client.DefaultRequestHeaders.UserAgent.ParseAdd( "sbox-text-to-animation/1.0" );
		return client;
	}

	HttpRangeSource( string url, long length ) { _url = url; Length = length; }
	public long Length { get; }

	public static async Task<HttpRangeSource> OpenAsync( string url, long expectedLength, CancellationToken token )
	{
		// a 1-byte range request both validates the URL and reports the total size
		using var request = new HttpRequestMessage( HttpMethod.Get, url );
		request.Headers.Range = new RangeHeaderValue( 0, 0 );
		using var response = await Http.SendAsync( request, HttpCompletionOption.ResponseHeadersRead, token );
		if ( response.StatusCode != System.Net.HttpStatusCode.PartialContent )
			throw new IOException( $"The download server didn't accept range requests ({(int)response.StatusCode})." );
		var total = response.Content.Headers.ContentRange?.Length ?? expectedLength;
		if ( expectedLength > 0 && total != expectedLength ) throw new IOException( $"Unexpected file size {total} (expected {expectedLength})." );
		return new HttpRangeSource( url, total );
	}

	public async Task<byte[]> ReadAsync( long offset, int count, CancellationToken token )
	{
		using var ms = new MemoryStream( count );
		await CopyAsync( offset, count, ms, null, token );
		return ms.ToArray();
	}

	public async Task CopyAsync( long offset, long count, Stream destination, Action<long>? progress, CancellationToken token )
	{
		long done = 0;
		for ( var attempt = 1; ; attempt++ )
		{
			try
			{
				using var request = new HttpRequestMessage( HttpMethod.Get, _url );
				request.Headers.Range = new RangeHeaderValue( offset + done, offset + count - 1 );
				using var stall = CancellationTokenSource.CreateLinkedTokenSource( token );
				stall.CancelAfter( StallTimeout );
				using var response = await Http.SendAsync( request, HttpCompletionOption.ResponseHeadersRead, stall.Token );
				if ( response.StatusCode != System.Net.HttpStatusCode.PartialContent )
					throw new IOException( $"Download failed: HTTP {(int)response.StatusCode}." );
				await using var body = await response.Content.ReadAsStreamAsync( stall.Token );
				var buffer = new byte[1 << 16];
				while ( done < count )
				{
					stall.CancelAfter( StallTimeout );
					var n = await body.ReadAsync( buffer.AsMemory( 0, (int)Math.Min( buffer.Length, count - done ) ), stall.Token );
					if ( n <= 0 ) break;
					await destination.WriteAsync( buffer.AsMemory( 0, n ), token );
					done += n;
					progress?.Invoke( done );
				}
				if ( done >= count ) return;
				throw new IOException( "The connection closed early." );
			}
			catch ( Exception e ) when ( e is IOException or HttpRequestException or OperationCanceledException && !token.IsCancellationRequested )
			{
				if ( attempt >= 4 ) throw new IOException( $"Download failed after {attempt} attempts: {e.Message}", e );
				await Task.Delay( TimeSpan.FromSeconds( 2 * attempt ), token );
			}
		}
	}
}

/// <summary>Central directory of a zip file read through ranges (Zip64 aware). Only STORED entries are readable.</summary>
public sealed class RemoteZip
{
	public sealed record Entry( string Name, long LocalHeaderOffset, long CompressedSize, long Size, uint Crc32, int Method );

	public IRangeSource Source { get; }
	public IReadOnlyDictionary<string, Entry> Entries { get; }

	RemoteZip( IRangeSource source, Dictionary<string, Entry> entries ) { Source = source; Entries = entries; }

	public static async Task<RemoteZip> OpenAsync( IRangeSource source, CancellationToken token )
	{
		var tailLength = (int)Math.Min( source.Length, 65536 + 22 );
		var tail = await source.ReadAsync( source.Length - tailLength, tailLength, token );
		var eocd = -1;
		for ( var i = tail.Length - 22; i >= 0; i-- )
			if ( BitConverter.ToUInt32( tail, i ) == 0x06054b50 ) { eocd = i; break; }
		if ( eocd < 0 ) throw new InvalidDataException( "Not a zip file (no end of central directory)." );
		long cdOffset = BitConverter.ToUInt32( tail, eocd + 16 );
		long cdSize = BitConverter.ToUInt32( tail, eocd + 12 );
		long count = BitConverter.ToUInt16( tail, eocd + 10 );
		if ( cdOffset == 0xFFFFFFFF || count == 0xFFFF || cdSize == 0xFFFFFFFF )
		{
			// Zip64: locator sits 20 bytes before the EOCD
			var loc = eocd - 20;
			if ( loc < 0 || BitConverter.ToUInt32( tail, loc ) != 0x07064b50 ) throw new InvalidDataException( "Zip64 locator missing." );
			var z64Offset = (long)BitConverter.ToUInt64( tail, loc + 8 );
			var z64 = await source.ReadAsync( z64Offset, 56, token );
			if ( BitConverter.ToUInt32( z64, 0 ) != 0x06064b50 ) throw new InvalidDataException( "Zip64 end record missing." );
			count = (long)BitConverter.ToUInt64( z64, 32 );
			cdSize = (long)BitConverter.ToUInt64( z64, 40 );
			cdOffset = (long)BitConverter.ToUInt64( z64, 48 );
		}
		if ( cdSize > 64 * 1024 * 1024 ) throw new InvalidDataException( "Zip central directory is too large." );
		var cd = await source.ReadAsync( cdOffset, (int)cdSize, token );
		var entries = new Dictionary<string, Entry>( StringComparer.Ordinal );
		var p = 0;
		for ( long e = 0; e < count; e++ )
		{
			if ( BitConverter.ToUInt32( cd, p ) != 0x02014b50 ) throw new InvalidDataException( "Corrupt zip central directory." );
			var method = BitConverter.ToUInt16( cd, p + 10 );
			var crc = BitConverter.ToUInt32( cd, p + 16 );
			long csize = BitConverter.ToUInt32( cd, p + 20 );
			long usize = BitConverter.ToUInt32( cd, p + 24 );
			var nameLen = BitConverter.ToUInt16( cd, p + 28 );
			var extraLen = BitConverter.ToUInt16( cd, p + 30 );
			var commentLen = BitConverter.ToUInt16( cd, p + 32 );
			long offset = BitConverter.ToUInt32( cd, p + 42 );
			var name = Encoding.UTF8.GetString( cd, p + 46, nameLen );
			// Zip64 extra field overrides 0xFFFFFFFF values in order: size, compressed size, offset
			var x = p + 46 + nameLen; var xEnd = x + extraLen;
			while ( x + 4 <= xEnd )
			{
				var id = BitConverter.ToUInt16( cd, x ); var len = BitConverter.ToUInt16( cd, x + 2 );
				if ( id == 1 )
				{
					var q = x + 4;
					if ( usize == 0xFFFFFFFF ) { usize = (long)BitConverter.ToUInt64( cd, q ); q += 8; }
					if ( csize == 0xFFFFFFFF ) { csize = (long)BitConverter.ToUInt64( cd, q ); q += 8; }
					if ( offset == 0xFFFFFFFF ) { offset = (long)BitConverter.ToUInt64( cd, q ); }
				}
				x += 4 + len;
			}
			entries[name] = new Entry( name, offset, csize, usize, crc, method );
			p += 46 + nameLen + extraLen + commentLen;
		}
		return new RemoteZip( source, entries );
	}

	/// <summary>Absolute offset of an entry's data (after its local header).</summary>
	public async Task<long> DataOffsetAsync( Entry entry, CancellationToken token )
	{
		var header = await Source.ReadAsync( entry.LocalHeaderOffset, 30, token );
		if ( BitConverter.ToUInt32( header, 0 ) != 0x04034b50 ) throw new InvalidDataException( "Corrupt zip local header." );
		return entry.LocalHeaderOffset + 30 + BitConverter.ToUInt16( header, 26 ) + BitConverter.ToUInt16( header, 28 );
	}

	public async Task<byte[]> ReadEntryAsync( Entry entry, CancellationToken token )
	{
		if ( entry.Method != 0 ) throw new NotSupportedException( "Only uncompressed zip entries can be read." );
		var data = await Source.ReadAsync( await DataOffsetAsync( entry, token ), checked((int)entry.Size), token );
		if ( Crc32.Compute( data ) != entry.Crc32 ) throw new InvalidDataException( $"{entry.Name} failed its CRC check." );
		return data;
	}
}

/// <summary>CRC-32 (IEEE), incremental.</summary>
public static class Crc32
{
	static readonly uint[] Table = Enumerable.Range( 0, 256 ).Select( n =>
	{
		var c = (uint)n;
		for ( var k = 0; k < 8; k++ ) c = (c & 1) != 0 ? 0xEDB88320u ^ (c >> 1) : c >> 1;
		return c;
	} ).ToArray();

	public static uint Compute( ReadOnlySpan<byte> data ) => Finish( Update( Start, data ) );
	public const uint Start = 0xFFFFFFFFu;
	public static uint Update( uint crc, ReadOnlySpan<byte> data )
	{
		foreach ( var b in data ) crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
		return crc;
	}
	public static uint Finish( uint crc ) => crc ^ 0xFFFFFFFFu;
}