Utility for downloading a single pinned model file with resumable transfers, disk-space checks, size and SHA256 verification, and progress reporting. It resumes partial files using HTTP range requests, writes to a .partial file, verifies checksum, and moves the final file into place.
using HumanoidMocap.Core.Inference;
using HumanoidMocap.EditorTools.Inference;
using System.Net;
using System.Net.Http.Headers;
using System.Security.Cryptography;
namespace HumanoidMocap.Worker;
/// <summary>One pinned model file: resumable, size- and checksum-verified. Model files run to several
/// gigabytes, so an interrupted transfer continues from its partial file instead of starting again,
/// disk space is checked first, and network failures say what to do. A partial that fails its
/// checksum is discarded; the destination only ever receives a verified file.</summary>
public static class ModelDownload
{
/// <summary>Megabytes or gigabytes, for "how much is left" in progress lines.</summary>
public static string Size(long bytes)=>bytes>=1_000_000_000?$"{bytes/1e9:0.0} GB":$"{(Math.Max(0,bytes)+999_999)/1_000_000} MB";
/// <param name="laterBytes">What still has to download after this file, so progress can say how much is left in all.</param>
public static async Task Fetch(HttpClient http,string url,string destination,long bytes,string sha256,Action<string> report,CancellationToken token,long laterBytes=0)
{
var name=Path.GetFileName(destination);var partial=destination+".partial";
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(destination))!);
var existing=File.Exists(partial)?new FileInfo(partial).Length:0;
if(existing>bytes){File.Delete(partial);existing=0;}
var root=Path.GetPathRoot(Path.GetFullPath(destination));
if(root is not null&&new DriveInfo(root) is {IsReady:true} drive&&drive.AvailableFreeSpace<bytes-existing+64L*1024*1024)
throw new IOException($"Not enough free space on {drive.Name} for {name}: {(bytes-existing)/1e9:0.0} GB needed, {drive.AvailableFreeSpace/1e9:0.0} GB free.");
if(existing<bytes)
{
try
{
using var request=new HttpRequestMessage(HttpMethod.Get,url);
if(existing>0)request.Headers.Range=new RangeHeaderValue(existing,null);
using var response=await http.SendAsync(request,HttpCompletionOption.ResponseHeadersRead,token);
// A server that ignores the range sends the whole file again.
if(existing>0&&response.StatusCode!=HttpStatusCode.PartialContent){existing=0;}
response.EnsureSuccessStatusCode();
if(response.Content.Headers.ContentLength is long length&&length!=bytes-existing)throw new InvalidDataException("Unexpected model download size for "+name+".");
// One format the editor reads for its progress bar: name, percent, and what is left in all.
string Line(long received)=>$"Downloading {name} · {received*100/bytes}% · {Size(bytes-received+laterBytes)} left";
report(Line(existing));
await using var source=await response.Content.ReadAsStreamAsync(token);
await using var output=new FileStream(partial,existing>0?FileMode.Append:FileMode.Create,FileAccess.Write,FileShare.None);
var buffer=new byte[1024*1024];var received=existing;var lastPercent=(int)(received*100/bytes);int count;
while((count=await source.ReadAsync(buffer,token))>0)
{
received+=count;if(received>bytes)throw new InvalidDataException("Model response exceeds its pinned size: "+name);
await output.WriteAsync(buffer.AsMemory(0,count),token);
var percent=(int)(received*100/bytes);if(percent>lastPercent){report(Line(received));lastPercent=percent;}
}
}
catch(Exception error) when(error is HttpRequestException or IOException&&error is not FileNotFoundException||error is TaskCanceledException&&!token.IsCancellationRequested)
{
// The partial file stays: the next attempt continues from it.
throw new IOException($"Could not finish downloading {name} ({bytes/1e9:0.00} GB). Check the internet connection and choose Retry; the download continues where it stopped. ({error.Message})",error);
}
}
if(new FileInfo(partial).Length!=bytes||!await Matches(partial,sha256,token))
{
File.Delete(partial);
throw new InvalidDataException($"{name} did not match its pinned checksum and was discarded. Choose Retry to download it again.");
}
File.Move(partial,destination);
// The checksum just verified travels with the file, so later jobs need not hash it again.
if(File.Exists(partial+".sha256"))File.Move(partial+".sha256",destination+".sha256",true);
}
public static Task<bool> Matches(string path,string sha256,CancellationToken token)=>Task.Run(()=>FileChecksum.Matches(path,sha256),token);
}