MediaBackend.cs
using System;
using Bimp.Resolver;
using Bimp.Resolver.Media;
namespace Bimp;
/// <summary>
/// Local playback of a single url or play token. Wraps the engine's <see cref="VideoPlayer"/> (video),
/// <see cref="MusicPlayer"/> (audio only, gives us a spectrum) or a <see cref="SegmentPlayer"/> (merged
/// YouTube media) behind one interface.
/// Each client owns one of these per <see cref="MediaPlayer"/>, it is never networked.
/// </summary>
public sealed class MediaBackend : IDisposable
{
VideoPlayer video;
MusicPlayer music;
/// <summary> The direct video player's frames (see <see cref="VideoFrameSink"/>). </summary>
readonly VideoFrameSink videoSink = new();
SegmentPlayer segments;
WebmSegmenter segmenter;
Resolver.Live.LiveStream live;
/// <summary>
/// Natively resolved media being prepared (resolving, downloading the first segment).
/// </summary>
Task<StreamSession> preparing;
string token;
bool audioOnly;
float startTime;
bool disposed;
/// <summary>
/// Merged media: how long from starting the segmenter to its first segment being ready (download + write),
/// averaged over every backend. A seek or join plans its first segment from where the synced time will be by then.
/// </summary>
static float firstSegmentSeconds = 1.5f;
RealTimeSince sinceSegmenter;
bool firstSegmentTimed;
/// <summary> Checking / fixing an MP4 before playing it (see <see cref="Mp4ChannelFix"/>). </summary>
bool checkingMp4;
string localDirectory;
readonly System.Threading.CancellationTokenSource cts = new();
/// <summary>
/// For streams that were started at an offset and whose clock starts at 0 - this is the offset.
/// </summary>
public float TimeOffset { get; private set; }
public string Url { get; private set; }
public bool IsAudioOnly => music is not null;
/// <summary>
/// Still resolving / downloading before anything can play.
/// </summary>
public bool IsPreparing => preparing is not null || checkingMp4;
/// <summary>
/// Something went wrong, user facing message.
/// </summary>
string error;
public string Error { get => error ?? live?.Error ?? segments?.Error; private set => error = value; }
/// <summary> Live: what we're connected to ("RTSP H.264 1920x1080 + G.711 mu-law"). </summary>
public string LiveDescription => live is null ? null
: $"{live.Description} | behind newest frame {LiveBehindNewest:0.00}s{(LiveLatency is { } l ? $", sent {l:0.00}s ago" : ", no sender clock")}{(live.Segmenter.Pcm is { } pcm ? $", audio {pcm.SampleRate}Hz {pcm.Channels}ch, A/V {pcm.Offset * 1000:+0;-0}ms ({pcm.State})" : "")}";
/// <summary> Live, decoded audio: its stream time minus the picture's (seconds, negative = audio late). </summary>
public double? LiveAvOffset => live?.Segmenter.Pcm?.Offset;
/// <summary>
/// Live: how long ago the frame on screen was sent, by the sender's clock (RTCP sender reports) - the latency
/// from the source's output to our screen. Null without sender reports (MPEG-TS, HLS, servers that send none).
/// </summary>
public double? LiveLatency => live?.Segmenter.WallClockZero is { } zero && segments is { Loaded: true }
? (DateTime.UtcNow - DateTime.UnixEpoch).TotalSeconds - (zero + segments.StreamTime)
: null;
/// <summary> Live: stream time from the frame on screen to the newest frame received. </summary>
public double LiveBehindNewest => live is null || segments is not { Loaded: true } ? 0 : live.Segmenter.NewestVideoTime - segments.StreamTime;
/// <summary> Merged media opened before this media time starts from the beginning (see <see cref="StartNative"/>). </summary>
const float FreshStartSeconds = 1.0f;
/// <summary>
/// Merged media: start at the keyframe before the planned time if it's at most this far behind it, else at the next
/// one (waiting for it). Stays inside the drift tolerance (<see cref="MediaPlayer"/>, 1.5 s) with the estimate's error.
/// </summary>
const float MaxStartLag = 1.0f;
/// <summary>
/// Where the synced timeline is, while it's moving (null while paused). Merged media that starts at a later keyframe
/// waits for it before opening its first segment.
/// </summary>
public float? SyncedTime { set { if ( segments is not null ) segments.SyncedTime = value; } }
/// <summary>
/// Playback has reached the end.
/// </summary>
bool finished;
public bool Finished { get => finished || (segments?.Finished ?? false); private set => finished = value; }
bool loaded;
float lastPlaybackTime;
/// <summary>
/// Has the media actually started (metadata loaded / audio is advancing)?
/// </summary>
public bool Loaded => segments?.Loaded ?? (!IsPreparing && (loaded || PlaybackTime > 0.01f));
/// <summary>
/// Time since this backend was created.
/// </summary>
public RealTimeSince SinceCreated { get; private set; }
/// <summary>
/// Position within the media, taking the start offset into account.
/// </summary>
public float Time => TimeOffset + PlaybackTime;
public float PlaybackTime => segments?.Time ?? video?.PlaybackTime ?? music?.PlaybackTime ?? 0;
/// <summary>
/// Duration as reported by the decoder.
/// </summary>
public float Duration => segments?.Duration ?? video?.Duration ?? music?.Duration ?? 0;
public bool IsPaused => segments?.IsPaused ?? video?.IsPaused ?? music?.Paused ?? false;
public Texture Texture => segments?.Texture ?? videoSink.Texture;
public int Width => segments?.Width ?? (video is null ? 0 : videoSink.Width);
public int Height => segments?.Height ?? (video is null ? 0 : videoSink.Height);
/// <summary>
/// Does the video have visible frames?
/// </summary>
public bool HasVideo => segments?.HasVideo ?? (video is not null && videoSink.Texture is not null && videoSink.Width > 4 && videoSink.Height > 4);
/// <summary>
/// 512 FFT values for visualizers. Empty for video playback.
/// </summary>
public ReadOnlySpan<float> Spectrum => music is not null ? music.Spectrum : ReadOnlySpan<float>.Empty;
public float Amplitude => music?.Amplitude ?? 0;
/// <summary>
/// The decoded audio format, e.g. "48000Hz 2ch" (for diagnostics). Empty until audio has started.
/// </summary>
public string AudioFormat
{
get
{
var (rate, ch) = segments is not null ? segments.AudioFormat : video is not null ? (video.SampleRate, video.Channels) : music is not null ? (music.SampleRate, music.Channels) : (0, 0);
return rate > 0 ? $"{rate}Hz {ch}ch" : "";
}
}
MediaBackend() { }
/// <summary>
/// Start playing <paramref name="url"/>. Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend Create( string url, bool audioOnly, float timeOffset )
{
var b = new MediaBackend
{
Url = url,
TimeOffset = timeOffset,
SinceCreated = 0,
};
b.StartUrl( url, audioOnly );
return b;
}
/// <summary>
/// Resolve a play token on this client and start playing it at <paramref name="startTime"/> (media seconds).
/// Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend CreateNative( string token, bool audioOnly, float startTime )
{
var b = new MediaBackend
{
Url = token,
SinceCreated = 0,
token = token,
audioOnly = audioOnly,
startTime = startTime,
};
b.preparing = StreamSessions.GetAsync( token );
return b;
}
/// <summary>
/// Connect to a live stream (RTSP tunnelled over HTTP, MPEG-TS over HTTP, HLS) and play it near its live edge.
/// Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend CreateLive( string url, int maxHeight )
{
var b = new MediaBackend { Url = url, SinceCreated = 0, liveMaxHeight = maxHeight };
try
{
// a live channel from an extractor (Twitch...): resolved on this client first (see StartNative)
if ( PlayToken.IsToken( url ) )
{
b.token = url;
b.preparing = StreamSessions.GetAsync( url );
return b;
}
b.StartLive( url, false );
}
catch ( Exception e )
{
b.Error = e.Message;
}
return b;
}
int liveMaxHeight = 720;
void StartLive( string url, bool hls )
{
live = new Resolver.Live.LiveStream( url, StreamSessions.NewCacheDirectory(), liveMaxHeight, hls );
segments = new SegmentPlayer( live.Segmenter );
}
/// <summary> The height a play token asks for (its pick, else the server's limit). </summary>
static int TokenHeight( string token, int fallback )
=> PlayToken.Parse( token ) is { } t ? (t.Height > 0 ? t.Height : t.MaxHeight) : fallback;
void StartUrl( string url, bool audio )
{
// MP4s whose header mislabels mono audio as stereo crackle in the engine - check first
if ( Mp4ChannelFix.Applies( url ) )
{
_ = StartMp4( url, audio );
return;
}
PlayUrlNow( url, audio );
}
async Task StartMp4( string url, bool audio )
{
checkingMp4 = true;
try
{
localDirectory = StreamSessions.NewCacheDirectory();
var path = await Mp4ChannelFix.PrepareAsync( url, localDirectory, cts.Token );
if ( disposed ) return;
if ( path is null ) PlayUrlNow( url, audio );
else PlayFile( path, audio );
}
catch ( OperationCanceledException )
{
}
catch ( Exception e )
{
if ( disposed ) return;
// couldn't fix it - play it as it is rather than not at all
Log.Warning( $"[bimp] Couldn't prepare {url}: {e.Message}" );
PlayUrlNow( url, audio );
}
finally
{
checkingMp4 = false;
}
}
void PlayFile( string path, bool audio )
{
try
{
if ( audio )
{
music = MusicPlayer.Play( FileSystem.Data, path );
music.OnFinished += () => Finished = true;
}
else
{
video = new VideoPlayer();
videoSink.Attach( video, null, null, "video" );
video.OnLoaded += () => loaded = true;
video.OnFinished += () => Finished = true;
video.Play( FileSystem.Data, path );
}
}
catch ( Exception e )
{
Error = e.Message;
Log.Warning( $"[bimp] Failed to play {path}: {e.Message}" );
}
}
void PlayUrlNow( string url, bool audio )
{
try
{
if ( audio )
{
music = MusicPlayer.PlayUrl( url );
music.OnFinished += () => Finished = true;
}
else
{
video = new VideoPlayer();
videoSink.Attach( video, null, null, "video" );
video.OnLoaded += () => loaded = true;
video.OnFinished += () => Finished = true;
video.Play( url );
}
}
catch ( Exception e )
{
Error = e.Message;
Log.Warning( $"[bimp] Failed to play {url}: {e.Message}" );
}
}
/// <summary>
/// The client side resolve finished - start the right kind of playback.
/// </summary>
void StartNative( StreamSession session )
{
var plan = session.Plan;
if ( plan.Kind == StreamKind.Direct )
{
// a progressive file - played straight from the site, seeked natively once loaded
StartUrl( plan.DirectUrl, audioOnly );
return;
}
if ( plan.Kind == StreamKind.Live )
{
liveMaxHeight = TokenHeight( token, liveMaxHeight );
StartLive( plan.DirectUrl, true );
return;
}
// The engine can only seek forward by decoding every frame in between, while the audio runs on - a 4K video
// can't decode much faster than it plays, so after a seek the picture crawled behind the sound for seconds. So
// merged media starts at a keyframe close behind where the synced time will be once the first segment is
// ready, or at the next one and waits there (see SegmentPlayer.SyncedTime) - it never seeks forward. The start
// of a video just plays from the beginning, a load time behind.
var planAhead = plan.Kind is StreamKind.Merge or StreamKind.Hls && !audioOnly && startTime >= FreshStartSeconds;
var plannedStart = planAhead ? startTime + (float)SinceCreated + firstSegmentSeconds : startTime;
var maxLag = planAhead ? MaxStartLag : float.MaxValue;
sinceSegmenter = 0;
if ( plan.Kind == StreamKind.Hls )
{
// a finished HLS playlist: remuxed from the segment at (or just after) the planned start, like merged media
live = new Resolver.Live.LiveStream( plan.DirectUrl, StreamSessions.NewCacheDirectory(), TokenHeight( token, 720 ), true, plannedStart, maxLag );
segments = new SegmentPlayer( live.Segmenter );
return;
}
segmenter = new WebmSegmenter( plan.Kind == StreamKind.Merge ? session.Video : null, session.Audio, plannedStart, StreamSessions.NewCacheDirectory(), maxLag, session.SwitchVideo );
if ( plan.Kind == StreamKind.Audio || audioOnly )
{
// one file from the cue point before the start time to the end, through MusicPlayer for the spectrum
_ = StartAudioFile();
return;
}
segments = new SegmentPlayer( segmenter );
}
async Task StartAudioFile()
{
preparing = null;
try
{
var segment = await segmenter.GetAsync( 0 );
if ( disposed || segment is null ) return;
music = MusicPlayer.Play( FileSystem.Data, segment.Path );
music.OnFinished += () => Finished = true;
}
catch ( Exception e )
{
if ( disposed ) return;
Error = e.Message;
StreamSessions.Forget( token );
}
}
/// <summary>
/// Call every frame. Pushes the latest video frame to the texture.
/// </summary>
public void Present()
{
if ( preparing is { IsCompleted: true } task )
{
preparing = null;
if ( task.IsCompletedSuccessfully )
{
try
{
StartNative( task.Result );
}
catch ( Exception e )
{
Error = e.Message;
StreamSessions.Forget( token );
}
}
else
{
var e = task.Exception?.InnerException;
Error = e is ResolveException ? e.Message : $"Couldn't load the stream: {e?.Message}";
Log.Warning( $"[bimp] Failed to resolve {token}: {e}" );
StreamSessions.Forget( token );
}
}
if ( !firstSegmentTimed && ((ISegmentFeed)segmenter ?? (live?.Segmenter is { IsLive: false } vod ? vod : null))?.TryGet( 0 ) is not null )
{
firstSegmentTimed = true;
firstSegmentSeconds = Math.Clamp( firstSegmentSeconds * 0.5f + sinceSegmenter * 0.5f, 0.3f, 10.0f );
}
segments?.Update();
if ( video is not null )
{
var presentStart = System.Diagnostics.Stopwatch.GetTimestamp();
video.Present();
MediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( presentStart ).TotalMilliseconds, "direct", 0 );
}
if ( live is not null && segments is not null )
{
live.PlayingSeq = segments.CurrentSeq;
// camera G.711 audio plays beside the video, lined up with the picture on screen
live.Segmenter.Pcm?.Update( segments.StreamTime, segments.Loaded && requestedPaused != true );
}
// MusicPlayer has no loaded event, so treat advancing time as loaded
var t = PlaybackTime;
if ( t > lastPlaybackTime + 0.01f ) loaded = true;
lastPlaybackTime = t;
}
bool? requestedPaused;
RealTimeSince sincePauseRequest;
/// <summary>
/// Pause or resume. Only calls into the native player when the requested state changes - not by
/// polling its IsPaused every frame, which can lag a call and made us call Pause/Resume repeatedly
/// (every extra Resume restarts the decoder's clock, which shows up as stutter).
/// If the native player still disagrees a while later (e.g. it hadn't started yet), ask once more.
/// </summary>
public void SetPaused( bool paused )
{
if ( requestedPaused == paused )
{
if ( IsPaused == paused || sincePauseRequest < 1.0f ) return;
}
requestedPaused = paused;
sincePauseRequest = 0;
MediaProbe.Note( paused ? "native Pause()" : "native Resume()" );
segments?.SetPaused( paused );
if ( video is not null )
{
if ( paused ) video.Pause(); else video.Resume();
}
if ( music is not null )
music.Paused = paused;
}
/// <summary>
/// Seek to a position within the media (not accounting for <see cref="TimeOffset"/> - pass media time).
/// </summary>
public void Seek( float mediaTime )
{
var local = MathF.Max( 0, mediaTime - TimeOffset );
MediaProbe.Note( $"native Seek({local:0.000}) from {PlaybackTime:0.000}" );
segments?.Seek( local );
video?.Seek( local );
music?.Seek( local );
Finished = false;
}
public void SetAudio( Vector3 position, bool spatial, float volume, float distance )
{
segments?.SetAudio( position, spatial, volume, distance );
live?.Segmenter.Pcm?.SetAudio( position, spatial, volume, distance );
// Non-spatial audio is played at the listener, slightly in front so it's centered
var pos = spatial ? position : Vector3.Forward * 64.0f;
if ( video is not null )
{
var a = video.Audio;
a.ListenLocal = !spatial;
a.Position = pos;
a.Volume = volume;
a.Distance = distance;
a.Falloff = MediaPlayer.AudioFalloff;
}
if ( music is not null )
{
music.ListenLocal = !spatial;
music.Position = pos;
music.Volume = volume;
music.Distance = distance;
music.Falloff = MediaPlayer.AudioFalloff;
}
}
public void Dispose()
{
disposed = true;
preparing = null;
cts.Cancel();
segments?.Dispose();
segments = null;
video?.Dispose();
video = null;
music?.Dispose();
music = null;
videoSink.Dispose();
// after the players, so the files aren't open anymore
segmenter?.Dispose();
segmenter = null;
live?.Dispose();
live = null;
if ( localDirectory is not null )
{
try
{
if ( FileSystem.Data.DirectoryExists( localDirectory ) ) FileSystem.Data.DeleteDirectory( localDirectory, true );
}
catch ( Exception e )
{
Log.Trace( $"[bimp] couldn't delete {localDirectory}: {e.Message}" );
}
localDirectory = null;
}
}
}