Code/Resolver/Media/RemoteFile.cs
using System.Globalization;
using System.Net.Http;
using System.Threading;
namespace Bimp.Resolver.Media;
public class FetchException : Exception
{
public int StatusCode { get; }
public FetchException( string message, int statusCode = 0 ) : base( message )
{
StatusCode = statusCode;
}
/// <summary> The server refused the url (expired, or tied to an old session) - worth re-resolving. </summary>
public bool IsRefused => StatusCode is 403 or 410;
}
/// <summary>
/// A remote file read with byte range requests. Large ranges are split into chunks because googlevideo
/// throttles very large single ranges (and unranged downloads) to playback speed.
/// </summary>
public sealed class RemoteFile
{
public const int ChunkSize = 8 * 1024 * 1024;
const int Retries = 3;
public string Url { get; private set; }
public long Size { get; private set; }
/// <summary>
/// Called once when the server refuses the url (403/410), to get a fresh one. Null to not retry.
/// </summary>
public Func<CancellationToken, Task<string>> Refresh { get; set; }
bool refreshed;
/// <summary> Bytes downloaded so far (YouTube stops serving some urls after so many - see WebmSegmenter). </summary>
public long BytesRead => System.Threading.Interlocked.Read( ref bytesRead );
long bytesRead;
public RemoteFile( string url, long size = 0 )
{
Url = url;
Size = size;
}
/// <summary>
/// Fill in <see cref="Size"/> (if not known) and return the first <paramref name="count"/> bytes.
/// </summary>
public async Task<byte[]> ProbeAsync( int count, CancellationToken ct )
{
var (data, total) = await RequestAsync( 0, count, ct );
if ( total > 0 ) Size = total;
if ( Size <= 0 ) throw new FetchException( "the server didn't report a file size" );
return data;
}
/// <summary>
/// Bytes [start, end) of the file.
/// </summary>
public async Task<byte[]> ReadAsync( long start, long end, CancellationToken ct )
{
if ( Size > 0 ) end = Math.Min( end, Size );
if ( end <= start ) return Array.Empty<byte>();
var result = new byte[end - start];
var pos = start;
while ( pos < end )
{
var want = (int)Math.Min( ChunkSize, end - pos );
var (chunk, _) = await RequestAsync( pos, want, ct );
if ( chunk.Length == 0 ) throw new FetchException( "the server returned no data" );
var n = Math.Min( chunk.Length, (int)(end - pos) );
Buffer.BlockCopy( chunk, 0, result, (int)(pos - start), n );
System.Threading.Interlocked.Add( ref bytesRead, n );
pos += n;
}
return result;
}
/// <summary>
/// Download the whole file into <paramref name="output"/>, chunk by chunk. <paramref name="patch"/> may modify
/// each chunk before it's written (given the chunk's offset in the file).
/// </summary>
public async Task CopyToAsync( System.IO.Stream output, Action<long, byte[]> patch, CancellationToken ct )
{
if ( Size <= 0 ) await ProbeAsync( 1, ct );
for ( long pos = 0; pos < Size; )
{
var want = (int)Math.Min( ChunkSize, Size - pos );
var (chunk, _) = await RequestAsync( pos, want, ct );
if ( chunk.Length == 0 ) throw new FetchException( "the server returned no data" );
patch?.Invoke( pos, chunk );
await output.WriteAsync( chunk, ct );
pos += chunk.Length;
}
}
async Task<(byte[] data, long total)> RequestAsync( long start, int count, CancellationToken ct )
{
for ( int attempt = 0; ; attempt++ )
{
ct.ThrowIfCancellationRequested();
int status;
try
{
var headers = new Dictionary<string, string> { ["Range"] = $"bytes={start}-{start + count - 1}" };
using var response = await Http.RequestAsync( Url, headers: headers, cancellationToken: ct );
status = (int)response.StatusCode;
if ( status is 200 or 206 )
{
var data = await response.Content.ReadAsByteArrayAsync( ct );
var total = TotalSize( response, status );
// A server that ignores Range sends the whole file
if ( status == 200 && start > 0 )
{
if ( data.Length <= start ) throw new FetchException( "the server ignored the byte range" );
data = data.AsSpan( (int)start, (int)Math.Min( count, data.Length - start ) ).ToArray();
}
else if ( data.Length > count )
{
data = data.AsSpan( 0, count ).ToArray();
}
return (data, total);
}
}
catch ( OperationCanceledException ) when ( ct.IsCancellationRequested )
{
throw;
}
catch ( FetchException )
{
throw;
}
catch ( Exception e ) when ( attempt < Retries )
{
// connection dropped - try again
Log.Trace( $"[bimp] range request failed ({e.Message}), retrying" );
await Task.Delay( 250 * (attempt + 1), ct );
continue;
}
if ( status is 403 or 410 && Refresh is not null && !refreshed )
{
refreshed = true;
Url = await Refresh( ct );
continue;
}
if ( status >= 500 && attempt < Retries )
{
await Task.Delay( 250 * (attempt + 1), ct );
continue;
}
throw new FetchException( $"the server returned {status}", status );
}
}
static long TotalSize( HttpResponseMessage response, int status )
{
var headers = response.Content.Headers;
if ( status == 206 )
{
// "bytes 0-1023/123456"
if ( headers.TryGetValues( "Content-Range", out var values ) )
{
var value = values.FirstOrDefault();
var slash = value?.LastIndexOf( '/' ) ?? -1;
if ( slash >= 0 && long.TryParse( value[(slash + 1)..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var total ) )
return total;
}
return 0;
}
return headers.ContentLength ?? 0;
}
}