Editor/SshCapture.cs
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using HumanoidMocapX.Core.Mapping;
using HumanoidMocapX.Core.Motion;
namespace HumanoidMocapX.EditorTools;
/// <summary>Captures on a processing machine over SSH with the Linux humanoid-mocap-worker:
/// uploads the video into a new job directory there, runs the worker, forwards its progress lines
/// and downloads the motion into the capture cache. The job directory is removed afterwards.
/// Cancelling ends the worker's session, and the machine then stops the worker.</summary>
// Starting ssh is an approved exception to the library's no-worker-process rule: the user asked
// for captures to run on their own GPU machine over SSH.
internal sealed class SshCapture(SshSettings settings, CaptureOptions options, string cache,
Action<string, double?> reportProgress = null) : IBodyCapture
{
internal const string ClientMissing = "Processing on another machine needs an SSH client: install OpenSSH so ssh is on PATH.";
/// <summary>Each job gets a new directory with this prefix; mkdir refuses an existing one.</summary>
internal const string JobRoot = "/tmp/humanoid-mocap-x-";
const string Result = "capture.hmotion";
const long MaximumResultBytes = 128L * 1024 * 1024;
// How long a cancelled worker may take to stop before the session is cut. The wrapper kills it after 10 s.
static readonly TimeSpan StopGrace = TimeSpan.FromSeconds(15);
static readonly TimeSpan CleanupTimeout = TimeSpan.FromSeconds(20);
// $1 job directory, $2 file name. A fresh mkdir fails if the path exists, so nothing is written through a planted link.
const string Upload = """umask 077 && mkdir -- "$1" && cat > "$1/$2" """;
// $1 job directory, then the worker command. The worker's stdin is /dev/null; the wrapper's own
// stdin is the session, so its end (cancel, editor exit or a dropped connection) sends TERM, then
// KILL while it still runs 10 s later. SIGPIPE is ignored so a dropped session cannot kill the
// wrapper before that. A failed or stopped job removes its directory; a finished one is left for
// the download.
const string Worker = """
trap "" PIPE
d=$1; shift
"$@" </dev/null &
p=$!
exec 3<&0
{ while read -r line <&3; do :; done; kill -TERM "$p"; n=0
while kill -0 "$p"; do [ "$n" -lt 10 ] || kill -KILL "$p"; sleep 1; n=$((n+1)); done; } >/dev/null 2>&1 &
k=$!
wait "$p"; c=$?
kill "$k" 2>/dev/null
[ "$c" -eq 0 ] || rm -rf -- "$d"
exit "$c"
""";
// $1 worker. Without arguments the worker prints its usage, which proves it starts.
const string Check = """
command -v ffmpeg >/dev/null && command -v ffprobe >/dev/null || { echo "FFmpeg is not installed there: ffmpeg and ffprobe must be on PATH." >&2; exit 1; }
usage=$("$1" </dev/null 2>&1)
case $usage in *"humanoid-mocap-worker capture"*) exit 0;; esac
echo "The worker at $1 did not start: $usage" >&2; exit 1
""";
const string Download = """cat -- "$1" """;
const string Remove = """rm -rf -- "$1" """;
int active;
public string UnavailableReason => ClientInstalled ? settings.Problem : ClientMissing;
static bool ClientInstalled => (Environment.GetEnvironmentVariable("PATH") ?? "").Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)
.Any(directory => File.Exists(Path.Combine(directory, OperatingSystem.IsWindows() ? "ssh.exe" : "ssh")));
public async Task<string> CaptureAsync(string video, double start, double? end, Action<string> progress, CancellationToken cancellationToken)
{
if (UnavailableReason is { } reason) throw new InvalidOperationException(reason);
var request = new HostedCaptureOptions(options.Model, options.CameraInterval, start, end); request.Validate();
var address = settings.Address; var worker = settings.Worker;
await using var input = new FileStream(video, FileMode.Open, FileAccess.Read, FileShare.Read, 65536, true);
if (Interlocked.CompareExchange(ref active, 1, 0) != 0)
throw new InvalidOperationException("Wait for the current capture on the processing machine to finish.");
var job = JobRoot + Guid.NewGuid().ToString("N");
void Report(string message, double? fraction) { progress?.Invoke(message); reportProgress?.Invoke(message, fraction); }
try
{
Report("Uploading video to " + address + "…", null);
var name = VideoName(video);
await RunAsync(address, Upload, [job, name], cancellationToken, input: input);
await RunAsync(address, Worker, [job, RemoteWorker(worker), .. WorkerArguments(job, name, request)], cancellationToken,
output: reader => ForwardAsync(reader, Report), grace: StopGrace);
Report("Downloading motion from " + address + "…", null);
return await Task.Run(() => DownloadAsync(address, job, video, cancellationToken), cancellationToken);
}
finally
{
try { await RemoveAsync(address, job, Report); }
finally { Volatile.Write(ref active, 0); }
}
}
/// <summary>Checks that the machine answers without a prompt, has FFmpeg and can start the
/// worker. Throws with the reason otherwise.</summary>
internal static Task CheckAsync(SshSettings settings, CancellationToken token)
{
if (!ClientInstalled) throw new InvalidOperationException(ClientMissing);
if (settings.Problem is { } problem) throw new ArgumentException(problem);
return RunAsync(settings.Address, Check, [RemoteWorker(settings.Worker)], token);
}
/// <summary>The worker's command line for one job; the video and output live in the job directory.</summary>
internal static List<string> WorkerArguments(string job, string video, HostedCaptureOptions request)
{
var arguments = new List<string> { "capture", job + "/" + video, job + "/" + Result,
"--start", request.Start.ToString("R", CultureInfo.InvariantCulture), "--model", request.Model == FlowModel.Base ? "base" : "latest",
"--camera-interval", request.CameraInterval.ToString(CultureInfo.InvariantCulture), "--cache", job };
if (request.End is { } stop) { arguments.Add("--end"); arguments.Add(stop.ToString("R", CultureInfo.InvariantCulture)); }
return arguments;
}
/// <summary>The remote file name: never the user's, which may hold quotes or spaces; a plain
/// extension is kept as FFmpeg's format hint.</summary>
internal static string VideoName(string video)
{
var extension = Path.GetExtension(video).ToLowerInvariant();
return "video" + (extension.Length is > 1 and <= 8 && extension[1..].All(char.IsAsciiLetterOrDigit) ? extension : "");
}
// ssh starts commands in the home directory, so ~/ is the same as ./ there.
internal static string RemoteWorker(string worker) => worker.StartsWith("~/", StringComparison.Ordinal) ? "./" + worker[2..] : worker;
/// <summary>ssh's arguments: never prompt (the editor cannot answer), give up on an unreachable
/// machine after 10 s or a silent one after a minute, and no terminal, so stderr stays separate.</summary>
internal static string[] Arguments(string address, string command)
=> ["-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=4", "-T", address, command];
/// <summary>The remote command line: sh runs <paramref name="script"/> with each argument quoted.</summary>
internal static string Command(string script, IEnumerable<string> arguments)
=> "sh -c " + Quote(script) + " humanoid-mocap-x " + string.Join(" ", arguments.Select(Quote));
/// <summary>Single-quotes <paramref name="text"/> for the login shell. Quotes and backslashes are
/// escaped outside the quotes, which POSIX shells and fish read the same way.</summary>
internal static string Quote(string text) => "'" + Regex.Replace(text, @"['\\]", match => "'\\" + match.Value + "'") + "'";
/// <summary>One worker progress line, <c>{"message":...,"fraction":...}</c>, or null for any other line.</summary>
internal static (string Message, double? Fraction)? Progress(string line)
{
try
{
using var json = JsonDocument.Parse(line);
var root = json.RootElement;
if (root.ValueKind != JsonValueKind.Object || !root.TryGetProperty("message", out var message) || message.ValueKind != JsonValueKind.String)
return null;
return (message.GetString(), root.TryGetProperty("fraction", out var fraction) && fraction.ValueKind == JsonValueKind.Number ? fraction.GetDouble() : null);
}
catch (JsonException) { return null; }
}
/// <summary>The last lines of a remote standard error without .NET stack frames, so the
/// worker's error line is what a failure shows.</summary>
internal static string Summary(string errors)
{
var lines = errors.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(line => !line.StartsWith("at ", StringComparison.Ordinal) && !line.StartsWith("--- ", StringComparison.Ordinal)).TakeLast(4);
var text = string.Join("\n", lines);
return text.Length == 0 ? "no error output" : text.Length > 1000 ? "…" + text[^1000..] : text;
}
static async Task ForwardAsync(StreamReader reader, Action<string, double?> report)
{
while (await reader.ReadLineAsync() is { } line)
if (Progress(line) is { } update) report(update.Message, update.Fraction);
}
async Task<string> DownloadAsync(string address, string job, string video, CancellationToken token)
{
Directory.CreateDirectory(cache);
var path = Path.Combine(cache, Guid.NewGuid().ToString("N") + ".hmotion"); var partial = path + ".partial";
try
{
await using (var output = new FileStream(partial, FileMode.CreateNew, FileAccess.Write, FileShare.None, 65536, true))
await RunAsync(address, Download, [job + "/" + Result], token, output: reader => CopyAsync(reader.BaseStream, output, token));
var motion = MotionDocument.Parse(await File.ReadAllBytesAsync(partial, token));
if (!motion.Bones.Any(bone => bone.Role == BoneRole.Hips)) throw new InvalidDataException("The processing machine must return full-body motion.");
// The worker only saw its own copy of the video; point the motion at the local file.
motion.Name = Path.GetFileNameWithoutExtension(video); motion.SourceVideo = video; motion.OriginalReconstruction = null;
await using (var output = new FileStream(partial, FileMode.Create, FileAccess.Write, FileShare.None, 65536, true))
await JsonSerializer.SerializeAsync(output, motion, MotionDocument.JsonOptions, token);
token.ThrowIfCancellationRequested(); File.Move(partial, path);
return path;
}
finally { if (File.Exists(partial)) File.Delete(partial); }
}
static async Task CopyAsync(Stream input, Stream output, CancellationToken token)
{
var buffer = new byte[65536]; long total = 0;
while (await input.ReadAsync(buffer, token) is var read && read > 0)
{
if ((total += read) > MaximumResultBytes) throw new InvalidDataException("The motion from the processing machine is larger than 128 MiB.");
await output.WriteAsync(buffer.AsMemory(0, read), token);
}
}
// A failed or stopped job's wrapper has already removed the directory; this covers the rest.
static async Task RemoveAsync(string address, string job, Action<string, double?> report)
{
using var timeout = new CancellationTokenSource(CleanupTimeout);
try { await RunAsync(address, Remove, [job], timeout.Token); }
catch (Exception error) when (error is InvalidOperationException or OperationCanceledException or IOException)
{ report("Could not remove " + job + " on the processing machine: " + error.Message, null); }
}
/// <summary>Runs <paramref name="script"/> on the machine until ssh exits. <paramref name="input"/>
/// is streamed to its standard input; otherwise standard input stays open, because the worker
/// wrapper treats its end as "stop". <paramref name="output"/> reads standard output. Cancelling
/// closes standard input and cuts the session after <paramref name="grace"/> (at once by default).
/// Throws with the remote standard error when ssh or the script fails.</summary>
static async Task RunAsync(string address, string script, IEnumerable<string> arguments, CancellationToken token,
Stream input = null, Func<StreamReader, Task> output = null, TimeSpan grace = default)
{
using var process = Start(address, Command(script, arguments));
var errors = process.StandardError.ReadToEndAsync(CancellationToken.None);
using var deadline = new CancellationTokenSource();
using var kill = deadline.Token.Register(() => Kill(process));
try
{
using (token.Register(() =>
{
if (grace <= TimeSpan.Zero) { deadline.Cancel(); return; }
Close(process.StandardInput); deadline.CancelAfter(grace);
}))
{
var sending = input is null ? Task.FromResult<Exception>(null) : SendAsync(input, process.StandardInput, token);
await (output?.Invoke(process.StandardOutput) ?? process.StandardOutput.BaseStream.CopyToAsync(Stream.Null));
await process.WaitForExitAsync(CancellationToken.None);
var unsent = await sending;
token.ThrowIfCancellationRequested();
if (process.ExitCode != 0)
throw new InvalidOperationException($"The processing machine failed (exit {process.ExitCode}): {Summary(await errors)}");
if (unsent is not null) throw new IOException("Could not upload the video: " + unsent.Message, unsent);
}
}
finally { Kill(process); }
}
static Process Start(string address, string command)
{
// UTF-8 explicitly: Windows would otherwise decode the worker's "·" and "…" with the console code page.
var info = new ProcessStartInfo("ssh")
{
RedirectStandardInput = true, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false,
StandardOutputEncoding = Encoding.UTF8, StandardErrorEncoding = Encoding.UTF8
};
foreach (var argument in Arguments(address, command)) info.ArgumentList.Add(argument);
try { return Process.Start(info) ?? throw new InvalidOperationException(ClientMissing); }
catch (Win32Exception error) { throw new InvalidOperationException(ClientMissing, error); }
}
// Returns the failure instead of throwing: if ssh failed as well, its error says more.
static async Task<Exception> SendAsync(Stream input, StreamWriter stdin, CancellationToken token)
{
try { await input.CopyToAsync(stdin.BaseStream, token); return null; }
catch (Exception error) when (error is IOException or OperationCanceledException or ObjectDisposedException) { return error; }
finally { Close(stdin); }
}
static void Close(StreamWriter stdin)
{
try { stdin.Close(); } catch (IOException) { }
}
static void Kill(Process process)
{
try { if (!process.HasExited) process.Kill(entireProcessTree: true); }
catch (Exception error) when (error is InvalidOperationException or Win32Exception) { }
}
}