Editor-side file synchronization utilities. Builds and caches a project file index with SHA-256 hashes, computes join plans comparing host and guest indexes, enforces chunk/window limits, assembles incoming file chunks into staged .part files, and safely accepts, writes, backs up, or deletes project asset files under a sanitized path.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
namespace TeamCreate;
/// <summary>
/// The fixed limits of project file synchronization. They are part of the protocol: a sender and a receiver must agree on the chunk size
/// (the assembler refuses a chunk that does not start on a chunk boundary) and on the window (the sender stops at eight unacknowledged chunks).
/// </summary>
public static class FileSyncLimits
{
/// <summary>Advertised by the host on welcome. A guest starts file synchronization only when its host holds this feature.</summary>
public const string Feature = "files-1";
/// <summary>256 KiB: about 350 KB as base64 JSON, far below Wire.MaxBytes, and small enough that one chunk never delays a live edit noticeably.</summary>
public const int ChunkSize = 256 * 1024;
/// <summary>
/// At most eight chunks per receiver are on the wire unacknowledged. The rest wait in the sender's own queue, outside the connection's FIFO
/// (Wire queues up to 256 frames in order and cannot reorder them), so a live edit or presence packet sent now is queued behind at most eight
/// chunks instead of behind the remainder of the file, and the receiver's 256-packet receive queue can never be flooded by one file.
/// </summary>
public const int Window = 8;
public const long MaxFileSize = 512L * 1024 * 1024;
public const int MaxEntries = 50000;
/// <summary>Index entries per files-index reply page (about 40 KB of JSON per page).</summary>
public const int IndexPage = 256;
}
public sealed record FileIndexEntry( string Path, string Sha256, long Size );
/// <summary>
/// Builds the file index: one entry for every path ProjectManifest.EnumerateInputs yields, hashed exactly as ProjectManifest hashes it
/// (a .meta file through MetaFileRules.Canonical). Digests are cached by (path, size, last-write time), so re-indexing an unchanged
/// project reads no file contents. A file above the transfer limit is listed with an empty digest instead of being read: it is never
/// transferred, and hashing half a gigabyte on the editor's frame would stall it.
/// </summary>
public sealed class FileSyncIndex
{
private readonly string root;
private readonly Dictionary<string, (long Size, long Stamp, string Hash)> cache = new( StringComparer.OrdinalIgnoreCase );
public FileSyncIndex( string root )
{
this.root = root;
}
/// <summary>How many files were actually read and hashed; the cache tests count it.</summary>
public int HashComputations { get; private set; }
/// <summary>Bytes read for hashing so far; the editor's poll bounds its per-frame work with it.</summary>
public long HashedBytes { get; private set; }
/// <summary>True when the last enumeration could not list a folder: a path missing from it is then unknown, not deleted.</summary>
public bool Incomplete { get; private set; }
/// <summary>The project-relative paths the manifest covers, refusing a project above the entry limit with a message that says so.</summary>
public List<string> Paths()
{
var paths = ProjectManifest.EnumerateInputs( root, out bool incomplete ).ToList();
Incomplete = incomplete;
if ( paths.Count > FileSyncLimits.MaxEntries )
{
throw new InvalidOperationException( $"The project has {paths.Count} files; file synchronization supports at most {FileSyncLimits.MaxEntries}. Synchronize this project through Git." );
}
return paths;
}
public List<FileIndexEntry> Build()
{
var entries = new List<FileIndexEntry>();
foreach ( string path in Paths() )
{
var entry = Entry( path );
if ( entry != null ) entries.Add( entry );
}
return entries;
}
/// <summary>The entry of one file, or null when it vanished between enumeration and reading (it is then simply not in this index).</summary>
public FileIndexEntry Entry( string path )
{
var info = new FileInfo( Path.Combine( root, path.Replace( '/', Path.DirectorySeparatorChar ) ) );
if ( !info.Exists ) return null;
if ( info.Length > FileSyncLimits.MaxFileSize ) return new FileIndexEntry( path, "", info.Length );
long stamp = info.LastWriteTimeUtc.Ticks;
if ( cache.TryGetValue( path, out var cached ) && cached.Size == info.Length && cached.Stamp == stamp )
{
return new FileIndexEntry( path, cached.Hash, cached.Size );
}
byte[] bytes;
try
{
bytes = File.ReadAllBytes( info.FullName );
}
catch ( FileNotFoundException )
{
return null;
}
catch ( DirectoryNotFoundException )
{
return null;
}
string hash = Hash( path, bytes );
HashComputations++;
HashedBytes += bytes.LongLength;
// The stamp is the one read before the content: a write that lands during the read moves the stamp again and is re-hashed next time.
cache[path] = (bytes.LongLength, stamp, hash);
return new FileIndexEntry( path, hash, bytes.LongLength );
}
public void Invalidate( string path )
{
if ( path != null ) cache.Remove( path );
}
public static string Hash( string path, byte[] bytes )
{
return Convert.ToHexString( SHA256.HashData( MetaFileRules.Canonical( path, bytes ) ) ).ToLowerInvariant();
}
}
/// <summary>What a joining guest does with the host's index: which files to fetch and what to tell the person about the rest.</summary>
public sealed record FileJoinPlanResult( List<FileIndexEntry> Fetch, string CodeNotice, List<string> Notices );
public static class FileJoinPlan
{
/// <summary>
/// Compares the two indexes through JoinReadiness.Evaluate. Missing and changed assets are fetched, except a scene open in the guest's
/// editor (the live scene snapshot owns it) and a file above the transfer limit (named in a notice). Code differences are never fetched;
/// they produce one notice. Guest-only files are left alone. A host entry that is malformed is refused by name.
/// </summary>
public static FileJoinPlanResult Plan( IReadOnlyList<FileIndexEntry> host, IReadOnlyList<FileIndexEntry> guest, IEnumerable<string> openScenes )
{
if ( host.Count > FileSyncLimits.MaxEntries )
{
throw new InvalidOperationException( $"The host's project has more than {FileSyncLimits.MaxEntries} files; file synchronization cannot continue. Synchronize this project through Git." );
}
var notices = new List<string>();
var remote = new Dictionary<string, FileIndexEntry>( StringComparer.OrdinalIgnoreCase );
foreach ( var entry in host )
{
// Only well-formedness here: code entries are compared, never written, and every asset is validated again by FileSyncReceiver
// (FilePathRules) before it is requested.
if ( string.IsNullOrWhiteSpace( entry?.Path ) || entry.Path.Length > 1024 || entry.Path.Any( char.IsControl ) || entry.Size < 0 || remote.ContainsKey( entry.Path ) )
{
notices.Add( $"The host listed an invalid project file entry \"{Bounded( entry?.Path )}\"; it was not transferred." );
continue;
}
remote[entry.Path] = entry;
}
var local = new Dictionary<string, string>( StringComparer.OrdinalIgnoreCase );
foreach ( var entry in guest ) local.TryAdd( entry.Path, entry.Sha256 ?? "" );
var readiness = JoinReadiness.Evaluate( Array.Empty<string>(), local, remote.ToDictionary( x => x.Key, x => x.Value.Sha256 ?? "", StringComparer.OrdinalIgnoreCase ) );
var open = new HashSet<string>( openScenes ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase );
var differing = readiness.Differences.Where( x => x.Kind != FileDifferenceKind.Extra ).ToList();
var code = differing.Where( x => x.FileKind == ProjectFileKind.Code ).Select( x => x.Path ).ToList();
string codeNotice = code.Count == 0 ? null
: $"Your code differs from the host's in {code.Count} file(s), for example {string.Join( ", ", code.Take( 3 ) )}. Update it through Git. Scenes and assets are synchronized automatically.";
var fetch = new List<FileIndexEntry>();
foreach ( var difference in differing.Where( x => x.FileKind == ProjectFileKind.Asset ) )
{
var entry = remote[difference.Path];
if ( open.Contains( entry.Path ) ) continue;
if ( entry.Size > FileSyncLimits.MaxFileSize )
{
notices.Add( TooLarge( entry.Path ) );
continue;
}
if ( !FileSyncReceiver.ValidHash( entry.Sha256 ) )
{
notices.Add( $"The host listed \"{Bounded( entry.Path )}\" without a valid SHA-256 digest; it was not transferred." );
continue;
}
fetch.Add( entry );
}
return new FileJoinPlanResult( fetch, codeNotice, notices );
}
public static string TooLarge( string path )
{
return $"The file \"{Bounded( path )}\" is larger than 512 MiB and is not synchronized. Copy it through Git or Asset Party.";
}
internal static string Bounded( string path )
{
if ( path == null ) return "";
string flat = path.Replace( '\r', ' ' ).Replace( '\n', ' ' );
return flat.Length <= 200 ? flat : flat.Substring( 0, 200 ) + "…";
}
}
/// <summary>Credits for chunks in flight to one receiver: a chunk is sent only with a credit, and the receiver's file-ack returns it.</summary>
public sealed class FileChunkWindow
{
public int InFlight { get; private set; }
public bool TryAcquire()
{
if ( InFlight >= FileSyncLimits.Window ) return false;
InFlight++;
return true;
}
public void Release()
{
if ( InFlight > 0 ) InFlight--;
}
}
/// <summary>
/// Assembles one incoming file from untrusted chunks into .collaboration/incoming/<sha>.part. A chunk must start on a chunk boundary,
/// be a whole chunk unless it is the last one, and lie within the declared size; anything else is refused without writing. A duplicate
/// chunk is accepted and ignored, so chunks may arrive in any order and be resent. The declared size is checked before the part file is
/// created, so a 600 MiB declaration writes nothing.
/// </summary>
public sealed class FileChunkAssembler : IDisposable
{
private readonly string part;
private readonly HashSet<long> received = new();
public FileChunkAssembler( string root, string path, string hash, long total, string partSuffix = null )
{
if ( !FileSyncReceiver.ValidPath( root, path, out _, out string error ) ) throw new InvalidOperationException( error );
if ( total < 0 || total > FileSyncLimits.MaxFileSize ) throw new InvalidOperationException( FileJoinPlan.TooLarge( path ) );
if ( !FileSyncReceiver.ValidHash( hash ) ) throw new InvalidOperationException( $"The file \"{FileJoinPlan.Bounded( path )}\" was offered without a valid SHA-256 digest." );
Path = path;
Sha256 = hash.ToLowerInvariant();
Total = total;
string incoming = System.IO.Path.Combine( root, ".collaboration", "incoming" );
Directory.CreateDirectory( incoming );
part = System.IO.Path.Combine( incoming, Sha256 + ( partSuffix ?? "" ) + ".part" );
using var file = new FileStream( part, FileMode.Create, FileAccess.Write, FileShare.None );
file.SetLength( total );
}
public string Path { get; }
public string Sha256 { get; }
public long Total { get; }
public long Received { get; private set; }
public bool IsComplete => Received == Total;
/// <summary>
/// The shape rule for one chunk of a transfer of <paramref name="total"/> bytes: it starts on a chunk boundary, lies within the declared size
/// and is a whole chunk unless it is the last. Shared with <see cref="BlobExchange"/>, which assembles geometry in memory by the same rule.
/// </summary>
public static bool IsValidChunk( long total, long offset, int length )
{
if ( offset < 0 || offset > total || length > FileSyncLimits.ChunkSize || length > total - offset ) return false;
if ( offset % FileSyncLimits.ChunkSize != 0 ) return false;
if ( length != FileSyncLimits.ChunkSize && offset + length != total ) return false;
return true;
}
public bool Add( long offset, byte[] data )
{
data ??= Array.Empty<byte>();
if ( !IsValidChunk( Total, offset, data.Length ) ) return false;
if ( !received.Add( offset ) ) return true;
using ( var file = new FileStream( part, FileMode.Open, FileAccess.Write, FileShare.None ) )
{
file.Position = offset;
file.Write( data );
}
Received += data.Length;
return true;
}
/// <summary>
/// Verifies the assembled file and hands it to the receiver's write procedure. Null on success (recovery then names the backup of
/// the replaced file, or null when nothing was replaced); otherwise the formal reason, and nothing was written.
/// </summary>
public string Complete( FileSyncReceiver receiver, out string recovery )
{
recovery = null;
if ( !IsComplete ) return $"The file \"{FileJoinPlan.Bounded( Path )}\" is incomplete.";
return receiver.AcceptStaged( Path, Sha256, part, out recovery );
}
public void Dispose()
{
try
{
if ( File.Exists( part ) ) File.Delete( part );
}
catch ( IOException )
{
}
catch ( UnauthorizedAccessException )
{
}
}
}
/// <summary>
/// The only code that writes or deletes project files on behalf of another machine. Every path goes through FilePathRules (normalisation,
/// traversal, drive, device names, junctions) and must classify as an asset: code and project settings are never written. A file that
/// exists and differs is first copied to .collaboration/recovery/files-<yyyyMMdd-HHmmss>/<path>; the new content is written to a
/// temporary file in the target's own folder and renamed over the target, so the target is never half-written.
/// </summary>
public sealed class FileSyncReceiver
{
private readonly string root;
private readonly Func<DateTime> clock;
public FileSyncReceiver( string root, Func<DateTime> clock = null )
{
this.root = root;
this.clock = clock ?? ( () => DateTime.UtcNow );
}
public static bool ValidHash( string hash )
{
return hash != null && hash.Length == 64 && hash.All( x => ( x >= '0' && x <= '9' ) || ( x >= 'a' && x <= 'f' ) || ( x >= 'A' && x <= 'F' ) );
}
public static bool ValidPath( string root, string path, out string target, out string error )
{
target = null;
// Classified first, so code is refused with the sentence that says why (FilePathRules would refuse it only as "not under Assets").
if ( !string.IsNullOrWhiteSpace( path ) && JoinReadiness.Classify( path ) != ProjectFileKind.Asset )
{
error = $"The file \"{FileJoinPlan.Bounded( path )}\" is code or project configuration and is never transferred. Update it through Git.";
return false;
}
if ( !FilePathRules.TryNormalize( path, out string normalized, out error ) )
{
error = $"The project file path \"{FileJoinPlan.Bounded( path )}\" was refused: {error}";
return false;
}
if ( !ProjectManifest.Covers( normalized ) )
{
error = $"The path \"{normalized}\" is outside the synchronized project files (Assets, excluding library and build folders) and was refused.";
return false;
}
if ( !FilePathRules.TryResolveUnder( root, normalized, out target, out error ) )
{
error = $"The project file path \"{normalized}\" was refused: {error}";
target = null;
return false;
}
return true;
}
/// <summary>Writes complete content received in one piece. Null on success, else the reason; nothing is written on failure.</summary>
public string Accept( string path, string hash, byte[] bytes, out string recovery )
{
recovery = null;
if ( bytes == null || bytes.LongLength > FileSyncLimits.MaxFileSize ) return FileJoinPlan.TooLarge( path );
if ( !ValidPath( root, path, out _, out string error ) ) return error;
string incoming = Path.Combine( root, ".collaboration", "incoming" );
Directory.CreateDirectory( incoming );
string part = Path.Combine( incoming, Guid.NewGuid().ToString( "N" ) + ".part" );
File.WriteAllBytes( part, bytes );
try
{
return AcceptStaged( path, hash, part, out recovery );
}
finally
{
if ( File.Exists( part ) ) File.Delete( part );
}
}
public string AcceptStaged( string path, string hash, string part, out string recovery )
{
recovery = null;
if ( !ValidPath( root, path, out string target, out string error ) ) return error;
if ( !ValidHash( hash ) ) return $"The file \"{path}\" was offered without a valid SHA-256 digest.";
byte[] content = File.ReadAllBytes( part );
if ( !string.Equals( FileSyncIndex.Hash( path, content ), hash, StringComparison.OrdinalIgnoreCase ) )
{
return $"The file \"{path}\" did not match its SHA-256 digest and was discarded.";
}
Directory.CreateDirectory( Path.GetDirectoryName( target )! );
// Resolved again after creating the folders: a junction planted in the path in between is caught here, before any write.
if ( !ValidPath( root, path, out target, out error ) ) return error;
if ( File.Exists( target ) )
{
if ( string.Equals( FileSyncIndex.Hash( path, File.ReadAllBytes( target ) ), hash, StringComparison.OrdinalIgnoreCase ) ) return null;
recovery = Backup( path, target );
}
// The ".tmp" extension is one ProjectManifest excludes, so a poll that runs mid-write never lists the temporary file as a project file.
string temp = Path.Combine( Path.GetDirectoryName( target )!, "." + Path.GetFileName( target ) + "." + Guid.NewGuid().ToString( "N" ) + ".tmp" );
try
{
File.WriteAllBytes( temp, content );
File.Move( temp, target, true );
}
finally
{
if ( File.Exists( temp ) ) File.Delete( temp );
}
return null;
}
/// <summary>Backs up, then deletes. Null on success (also when the file is already gone), else the reason.</summary>
public string Delete( string path, out string recovery )
{
recovery = null;
if ( !ValidPath( root, path, out string target, out string error ) ) return error;
if ( !File.Exists( target ) ) return null;
recovery = Backup( path, target );
File.Delete( target );
return null;
}
// A copy is never overwritten: two replacements of one file within the same second get distinct folders.
private string Backup( string path, string target )
{
string stamp = clock().ToString( "yyyyMMdd-HHmmss", System.Globalization.CultureInfo.InvariantCulture );
string relative = ".collaboration/recovery/files-" + stamp + "/" + path;
for ( int n = 2; File.Exists( Path.Combine( root, relative.Replace( '/', Path.DirectorySeparatorChar ) ) ); n++ )
{
relative = ".collaboration/recovery/files-" + stamp + "-" + n + "/" + path;
}
string backup = Path.Combine( root, relative.Replace( '/', Path.DirectorySeparatorChar ) );
Directory.CreateDirectory( Path.GetDirectoryName( backup )! );
File.Copy( target, backup );
return relative;
}
}