A content-addressed blob storage used by the editor collaboration system. It stores geometry bytes keyed by lowercase SHA-256, keeps an LRU in-memory cache, optional disk cache under .collaboration/blobs, validates hashes, enforces size limits and prunes disk cache.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
namespace TeamCreate;
/// <summary>
/// The fixed limits of mapping-tool geometry synchronization (docs/BLOB-SYNC.md). They are part of the protocol: a receiver refuses what
/// exceeds them, so a sender that respected them is never refused for size.
/// </summary>
public static class BlobLimits
{
/// <summary>Advertised by the host on welcome. A guest sends geometry placeholders only to a host that holds this feature.</summary>
public const string Feature = "blobs-1";
/// <summary>The largest geometry (decoded bytes) that is synchronized. The largest mesh in the installed sample scenes is 576 KB.</summary>
public const int MaxBlobBytes = 32 * 1024 * 1024;
/// <summary>Geometry that has started to arrive and is not complete yet, summed over one endpoint (every member on a host).</summary>
public const long MaxPendingBytes = 64L * 1024 * 1024;
/// <summary>Scene packets one endpoint holds while the geometry they name is on its way.</summary>
public const int MaxPendingPackets = 32;
/// <summary>How long a held packet waits for its geometry before it is discarded and the scene is synchronized again.</summary>
public const int WaitSeconds = 60;
/// <summary>Hashes one blob-need may name; the remainder of a longer request is ignored.</summary>
public const int MaxHashesPerRequest = 64;
/// <summary>
/// Shortest text value that is moved out of an object's state (4 KiB). Below this the inline text costs less than the placeholder
/// round trip, and nothing in the engine writes geometry that small except an empty mesh.
/// </summary>
public const int MinExternalChars = 4096;
/// <summary>One new version of an object's geometry per this many milliseconds, so dragging a vertex does not flood the session.</summary>
public const int VersionIntervalMs = 500;
public const long MemoryCacheBytes = 256L * 1024 * 1024;
public const long DiskCacheBytes = 256L * 1024 * 1024;
}
/// <summary>
/// Content-addressed geometry bytes, keyed by the lowercase hex SHA-256 of the bytes. A bounded in-memory cache (least recently used out)
/// sits in front of an optional disk cache in <c>.collaboration/blobs/<sha256></c>. Nothing is stored under a key its bytes do not
/// hash to, and a key that is not exactly 64 lowercase hex digits never reaches the file system, so a hostile peer cannot name a path.
/// </summary>
public sealed class BlobStore
{
private sealed class Entry
{
public byte[] Bytes;
public long Touched;
}
private readonly object gate = new();
private readonly Dictionary<string, Entry> memory = new( StringComparer.Ordinal );
private readonly long memoryCap;
private long diskCap;
private string directory;
private long clock;
public BlobStore( string root, long memoryCap, long diskCap )
{
this.memoryCap = memoryCap;
if ( root != null ) AttachDisk( root, diskCap );
}
public int MemoryCount
{
get { lock ( gate ) return memory.Count; }
}
public long MemoryBytes { get; private set; }
public static string HashOf( byte[] bytes )
{
return Convert.ToHexString( SHA256.HashData( bytes ) ).ToLowerInvariant();
}
public static bool ValidHash( string hash )
{
if ( hash == null || hash.Length != 64 ) return false;
foreach ( char c in hash )
{
if ( !( ( c >= '0' && c <= '9' ) || ( c >= 'a' && c <= 'f' ) ) ) return false;
}
return true;
}
/// <summary>
/// Starts keeping blobs on disk under <paramref name="root"/>, and writes the ones already in memory, so an object captured before the
/// session began hosting can still be served to a late joiner. A folder that cannot be created leaves the store memory-only.
/// </summary>
public void AttachDisk( string root, long cap )
{
string folder;
try
{
folder = Path.Combine( root, ".collaboration", "blobs" );
Directory.CreateDirectory( folder );
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException )
{
return;
}
lock ( gate )
{
directory = folder;
diskCap = cap;
foreach ( KeyValuePair<string, Entry> pair in memory ) WriteFile( pair.Key, pair.Value.Bytes );
}
}
/// <summary>Stores <paramref name="bytes"/> under their own hash. Null (and <paramref name="error"/>) when they are empty or over the limit.</summary>
public string Put( byte[] bytes, out string error )
{
error = null;
if ( bytes == null || bytes.Length == 0 )
{
error = "Empty geometry is not stored.";
return null;
}
if ( bytes.Length > BlobLimits.MaxBlobBytes )
{
error = TooLarge();
return null;
}
string hash = HashOf( bytes );
return Store( hash, bytes ) ? hash : null;
}
/// <summary>Stores <paramref name="bytes"/> only if they hash to <paramref name="hash"/>; anything else is refused and nothing is written.</summary>
public bool Put( string hash, byte[] bytes, out string error )
{
error = null;
if ( !ValidHash( hash ) )
{
error = "The geometry was offered under a name that is not a SHA-256 digest.";
return false;
}
if ( bytes == null || bytes.Length == 0 )
{
error = "Empty geometry is not stored.";
return false;
}
if ( bytes.Length > BlobLimits.MaxBlobBytes )
{
error = TooLarge();
return false;
}
if ( HashOf( bytes ) != hash )
{
error = "The geometry did not match the digest it was offered under and was discarded.";
return false;
}
return Store( hash, bytes );
}
/// <summary>
/// Records bytes the caller has just hashed itself (<see cref="BlobRules.Externalize"/> does, to name them), without hashing again. This runs for every
/// mesh on every scan, so a blob already in memory is only touched: no copy, no disk check.
/// </summary>
public void AddTrusted( string hash, byte[] bytes )
{
lock ( gate )
{
if ( memory.TryGetValue( hash, out Entry known ) )
{
known.Touched = ++clock;
return;
}
if ( bytes.Length > 0 && bytes.Length <= BlobLimits.MaxBlobBytes ) Store( hash, bytes );
}
}
public static string TooLarge() => "The geometry is larger than the 32 MiB collaboration limit and was not synchronized.";
public bool Has( string hash )
{
if ( !ValidHash( hash ) ) return false;
lock ( gate )
{
if ( memory.ContainsKey( hash ) ) return true;
return directory != null && File.Exists( Path.Combine( directory, hash ) );
}
}
public bool TryGet( string hash, out byte[] bytes )
{
bytes = null;
if ( !ValidHash( hash ) ) return false;
lock ( gate )
{
if ( memory.TryGetValue( hash, out Entry entry ) )
{
entry.Touched = ++clock;
bytes = entry.Bytes;
return true;
}
if ( directory == null ) return false;
string file = Path.Combine( directory, hash );
byte[] read;
try
{
if ( !File.Exists( file ) ) return false;
read = File.ReadAllBytes( file );
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException )
{
return false;
}
// A file that does not hash to its own name (a torn write, a disk fault, a person's edit) is never served: it is removed so
// the geometry is fetched again.
if ( read.Length == 0 || read.Length > BlobLimits.MaxBlobBytes || HashOf( read ) != hash )
{
TryDelete( file );
return false;
}
Remember( hash, read );
try
{
File.SetLastWriteTimeUtc( file, DateTime.UtcNow );
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException )
{
}
bytes = read;
return true;
}
}
/// <summary>Deletes the oldest files until the disk cache is within its cap. Called by the session from time to time, never on the capture path.</summary>
public void Prune()
{
lock ( gate )
{
if ( directory == null || diskCap <= 0 ) return;
List<FileInfo> files;
try
{
files = new DirectoryInfo( directory ).GetFiles().Where( x => ValidHash( x.Name ) ).ToList();
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException )
{
return;
}
long total = files.Sum( x => x.Length );
foreach ( FileInfo file in files.OrderBy( x => x.LastWriteTimeUtc ) )
{
if ( total <= diskCap ) break;
total -= file.Length;
TryDelete( file.FullName );
}
}
}
private bool Store( string hash, byte[] bytes )
{
lock ( gate )
{
if ( memory.TryGetValue( hash, out Entry known ) )
{
known.Touched = ++clock;
if ( directory != null && !File.Exists( Path.Combine( directory, hash ) ) ) WriteFile( hash, bytes );
return true;
}
Remember( hash, bytes );
if ( directory != null ) WriteFile( hash, bytes );
return true;
}
}
// Adds to the memory cache and evicts the least recently used entries until it fits. The entry just added is never the one evicted.
private void Remember( string hash, byte[] bytes )
{
memory[hash] = new Entry { Bytes = bytes, Touched = ++clock };
MemoryBytes += bytes.Length;
while ( MemoryBytes > memoryCap && memory.Count > 1 )
{
string oldest = null;
long oldestTouch = long.MaxValue;
foreach ( KeyValuePair<string, Entry> pair in memory )
{
if ( pair.Key != hash && pair.Value.Touched < oldestTouch )
{
oldest = pair.Key;
oldestTouch = pair.Value.Touched;
}
}
if ( oldest == null ) break;
MemoryBytes -= memory[oldest].Bytes.Length;
memory.Remove( oldest );
}
}
// Written to a temporary name and renamed, so a reader never sees half a file. A failure leaves the memory copy as the only one.
private void WriteFile( string hash, byte[] bytes )
{
string target = Path.Combine( directory, hash );
string temporary = Path.Combine( directory, hash + "." + Guid.NewGuid().ToString( "N" ) + ".tmp" );
try
{
File.WriteAllBytes( temporary, bytes );
File.Move( temporary, target, true );
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException )
{
TryDelete( temporary );
}
}
private static void TryDelete( string path )
{
try
{
File.Delete( path );
}
catch ( Exception ex ) when ( ex is IOException or UnauthorizedAccessException )
{
}
}
}