Code/Resolver/Extractors/SiteExtractors.cs
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
using System.Threading;
namespace Bimp.Resolver.Extractors;
/// <summary>
/// twitch.tv/{channel} (live), twitch.tv/videos/{id} (VOD), twitch.tv/{channel}/clip/{slug} and clips.twitch.tv/{slug}.
/// Uses the web player's public GQL client id and playback access tokens, then the usher HLS playlists (H.264 TS).
/// </summary>
public sealed class TwitchExtractor : SiteExtractor
{
public override string Key => "twitch";
protected override string[] Hosts => new[] { "twitch.tv" };
const string Gql = "https://gql.twitch.tv/gql";
const string ClientId = "kimne78kx3ncx6brgo4mv6wki5h1ko";
static readonly Dictionary<string, string> Headers = new() { ["Client-ID"] = ClientId };
static readonly string[] NotChannels = { "directory", "videos", "settings", "search", "downloads", "p", "jobs", "turbo", "subscriptions", "inventory", "wallet", "friends", "messages" };
static readonly Regex Login = new( "^[a-z0-9_]{2,25}$", RegexOptions.IgnoreCase | RegexOptions.Compiled );
/// <summary> "live:{login}", "vod:{id}" or "clip:{slug}". </summary>
protected override string ParseId( Uri url )
{
var p = url.AbsolutePath.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
var host = url.Host.ToLowerInvariant();
if ( host.StartsWith( "clips." ) ) return p.Length >= 1 ? $"clip:{p[0]}" : null;
if ( p.Length >= 2 && p[0] == "videos" && long.TryParse( p[1], out _ ) ) return $"vod:{p[1]}";
if ( p.Length >= 3 && p[1] == "clip" ) return $"clip:{p[2]}";
if ( p.Length >= 3 && p[1] == "video" && long.TryParse( p[2], out _ ) ) return $"vod:{p[2]}";
if ( p.Length >= 1 && Login.IsMatch( p[0] ) && !NotChannels.Contains( p[0].ToLowerInvariant() ) ) return $"live:{p[0].ToLowerInvariant()}";
return null;
}
protected override async Task<SiteMedia> FetchAsync( string id, CancellationToken ct )
{
var kind = id[..id.IndexOf( ':' )];
var value = id[(kind.Length + 1)..];
return kind switch
{
"live" => await Live( value, ct ),
"vod" => await Vod( value, ct ),
_ => await Clip( value, ct ),
};
}
static Task<JsonNode> Query( string query, JsonObject variables, CancellationToken ct )
=> PostJson( Gql, new JsonObject { ["query"] = query, ["variables"] = variables }, ct, Headers );
static async Task<SiteMedia> Live( string login, CancellationToken ct )
{
var j = await Query( "query($login: String!) { user(login: $login) { displayName stream { title } } streamPlaybackAccessToken(channelName: $login, params: {platform: \"web\", playerBackend: \"mediaplayer\", playerType: \"site\"}) { value signature } }",
new JsonObject { ["login"] = login }, ct );
var user = j?["data"]?["user"];
if ( user is null ) throw new ResolveException( "That Twitch channel doesn't exist." );
if ( user["stream"] is not JsonObject stream ) throw new ResolveException( $"{user["displayName"].Str() ?? login} isn't live right now." );
var token = j["data"]["streamPlaybackAccessToken"];
return new SiteMedia
{
Title = $"{user["displayName"].Str()}: {stream["title"].Str()}",
IsLive = true,
Hls = $"https://usher.ttvnw.net/api/channel/hls/{login}.m3u8?{Usher( token )}",
};
}
static async Task<SiteMedia> Vod( string id, CancellationToken ct )
{
var j = await Query( "query($id: ID!) { video(id: $id) { title lengthSeconds owner { displayName } } videoPlaybackAccessToken(id: $id, params: {platform: \"web\", playerBackend: \"mediaplayer\", playerType: \"site\"}) { value signature } }",
new JsonObject { ["id"] = id }, ct );
var video = j?["data"]?["video"] ?? throw new ResolveException( "That Twitch video doesn't exist (or is subscriber only)." );
return new SiteMedia
{
Title = $"{video["owner"]?["displayName"].Str()}: {video["title"].Str()}",
Duration = video["lengthSeconds"].Float(),
Hls = $"https://usher.ttvnw.net/vod/{id}.m3u8?{Usher( j["data"]["videoPlaybackAccessToken"] )}",
};
}
static async Task<SiteMedia> Clip( string slug, CancellationToken ct )
{
var j = await Query( "query($slug: ID!) { clip(slug: $slug) { title durationSeconds broadcaster { displayName } videoQualities { quality sourceURL } playbackAccessToken(params: {platform: \"web\", playerBackend: \"mediaplayer\", playerType: \"site\"}) { value signature } } }",
new JsonObject { ["slug"] = slug }, ct );
var clip = j?["data"]?["clip"] ?? throw new ResolveException( "That Twitch clip doesn't exist." );
var token = clip["playbackAccessToken"];
var query = $"sig={token?["signature"].Str()}&token={Uri.EscapeDataString( token?["value"].Str() ?? "" )}";
var m = new SiteMedia { Title = $"{clip["broadcaster"]?["displayName"].Str()}: {clip["title"].Str()}", Duration = clip["durationSeconds"].Float() };
foreach ( var q in clip["videoQualities"].Items() )
{
var url = q["sourceURL"].Str();
if ( string.IsNullOrEmpty( url ) ) continue;
m.Files.Add( new MediaFormat { Id = q["quality"].Str(), Url = $"{url}?{query}", Container = "mp4", VideoCodec = "h264", AudioCodec = "aac", Height = q["quality"].Int() } );
}
return m;
}
static string Usher( JsonNode token )
{
if ( token?["value"].Str() is not { } value ) throw new ResolveException( "Twitch didn't hand out a playback token." );
return $"sig={token["signature"].Str()}&token={Uri.EscapeDataString( value )}&allow_source=true&allow_audio_only=true&player=twitchweb&p={Random.Shared.Next( 1000000, 9999999 )}";
}
}
/// <summary>
/// kick.com/{channel} (live) and kick.com/{channel}/videos/{uuid} (VOD): the public channel and video APIs, then
/// their HLS (Amazon IVS, H.264 TS).
/// </summary>
public sealed class KickExtractor : SiteExtractor
{
public override string Key => "kick";
protected override string[] Hosts => new[] { "kick.com" };
static readonly Regex Slug = new( "^[a-z0-9_-]{2,40}$", RegexOptions.IgnoreCase | RegexOptions.Compiled );
static readonly Regex Uuid = new( "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", RegexOptions.IgnoreCase | RegexOptions.Compiled );
static readonly string[] NotChannels = { "categories", "category", "search", "following", "browse", "dashboard", "terms-of-service", "privacy-policy" };
/// <summary> "live:{slug}" or "vod:{uuid}". </summary>
protected override string ParseId( Uri url )
{
var p = url.AbsolutePath.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
var video = p.FirstOrDefault( s => Uuid.IsMatch( s ) );
if ( video is not null && p.Any( s => s is "videos" or "video" ) ) return $"vod:{video}";
if ( p.Length == 1 && Slug.IsMatch( p[0] ) && !NotChannels.Contains( p[0].ToLowerInvariant() ) ) return $"live:{p[0].ToLowerInvariant()}";
return null;
}
protected override async Task<SiteMedia> FetchAsync( string id, CancellationToken ct )
{
var value = id[(id.IndexOf( ':' ) + 1)..];
if ( id.StartsWith( "vod:" ) )
{
var v = await GetJson( $"https://kick.com/api/v1/video/{value}", ct );
var source = v?["source"].Str() ?? throw new ResolveException( "That Kick video can't be played." );
return new SiteMedia
{
Title = v["livestream"]?["session_title"].Str() ?? "Kick video",
Duration = (v["livestream"]?["duration"]).Long() / 1000f,
Hls = source,
};
}
var c = await GetJson( $"https://kick.com/api/v2/channels/{value}", ct );
if ( c?["livestream"] is not JsonObject live || c["playback_url"].Str() is not { } playback )
throw new ResolveException( $"{c?["user"]?["username"].Str() ?? value} isn't live right now." );
return new SiteMedia { Title = $"{c["user"]?["username"].Str() ?? value}: {live["session_title"].Str()}", IsLive = true, Hls = playback };
}
}
/// <summary> x.com / twitter.com/{user}/status/{id}: the embed (syndication) API's mp4 variants. </summary>
public sealed class XExtractor : SiteExtractor
{
public override string Key => "x";
protected override string[] Hosts => new[] { "x.com", "twitter.com", "fxtwitter.com", "vxtwitter.com" };
static readonly Regex Status = new( "/status(?:es)?/(\\d{5,25})", RegexOptions.Compiled );
static readonly Regex Size = new( "/(\\d{2,5})x(\\d{2,5})/", RegexOptions.Compiled );
protected override string ParseId( Uri url )
{
var m = Status.Match( url.AbsolutePath );
return m.Success ? m.Groups[1].Value : null;
}
protected override async Task<SiteMedia> FetchAsync( string id, CancellationToken ct )
{
// any token works; it's just required
var j = await GetJson( $"https://cdn.syndication.twimg.com/tweet-result?id={id}&token=a&lang=en", ct );
if ( j?["__typename"].Str() == "TweetTombstone" ) throw new ResolveException( "That post isn't available." );
var video = j?["mediaDetails"].Items().FirstOrDefault( d => d["video_info"] is not null )?["video_info"]
?? throw new ResolveException( "That post has no video." );
var text = j["text"].Str() ?? "";
var m = new SiteMedia
{
Title = $"{j["user"]?["name"].Str()}: {(text.Length > 80 ? text[..80] + "..." : text)}",
Duration = video["duration_millis"].Long() / 1000f,
};
foreach ( var v in video["variants"].Items() )
{
var url = v["url"].Str();
if ( v["content_type"].Str() == "application/x-mpegURL" ) { m.Hls ??= url; continue; }
if ( v["content_type"].Str() != "video/mp4" || url is null ) continue;
var size = Size.Match( url );
m.Files.Add( new MediaFormat { Id = url, Url = url, Container = "mp4", VideoCodec = "h264", AudioCodec = "aac", Height = size.Success ? int.Parse( size.Groups[2].Value ) : 0, Bitrate = v["bitrate"].Long() } );
}
return m;
}
}
/// <summary> archive.org/details/{id}: the item's best mp4 / webm (or, for audio items, mp3 / ogg) file. </summary>
public sealed class ArchiveExtractor : SiteExtractor
{
public override string Key => "archive";
protected override string[] Hosts => new[] { "archive.org" };
static readonly string[] VideoExtensions = { ".mp4", ".webm", ".m4v" };
static readonly string[] AudioExtensions = { ".mp3", ".ogg", ".m4a", ".flac", ".wav" };
/// <summary> "{item}" or "{item}/{file}". </summary>
protected override string ParseId( Uri url )
{
var p = url.AbsolutePath.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
if ( p.Length < 2 || p[0] is not ("details" or "embed" or "download") ) return null;
return p.Length >= 3 && p[0] == "download" ? $"{p[1]}/{string.Join( '/', p.Skip( 2 ).Select( Uri.UnescapeDataString ) )}" : p[1];
}
protected override async Task<SiteMedia> FetchAsync( string id, CancellationToken ct )
{
var item = id.Split( '/' )[0];
var wanted = id.Length > item.Length ? id[(item.Length + 1)..] : null;
var j = await GetJson( $"https://archive.org/metadata/{item}", ct );
if ( j?["files"] is not JsonArray files ) throw new ResolveException( "That archive.org item doesn't exist (or is dark)." );
var candidates = files.Select( f => (name: f?["name"].Str(), size: f?["size"].Long() ?? 0, length: Length( f?["length"].Str() ), height: f?["height"].Int() ?? 0) )
.Where( f => f.name is not null && (wanted is null || f.name == wanted) )
.ToList();
string Ext( string n ) => System.IO.Path.GetExtension( n ).ToLowerInvariant();
var videos = candidates.Where( f => VideoExtensions.Contains( Ext( f.name ) ) ).ToList();
var audio = candidates.Where( f => AudioExtensions.Contains( Ext( f.name ) ) ).ToList();
var m = new SiteMedia { Title = j["metadata"]?["title"].Str() ?? item };
var chosen = videos.Count > 0 ? videos : audio;
if ( chosen.Count == 0 ) throw new ResolveException( "That archive.org item has no video or audio file the engine can play." );
m.AudioOnly = videos.Count == 0;
foreach ( var f in chosen )
{
var url = $"https://archive.org/download/{item}/{string.Join( '/', f.name.Split( '/' ).Select( Uri.EscapeDataString ) )}";
var ext = Ext( f.name ).TrimStart( '.' );
m.Files.Add( new MediaFormat { Id = f.name, Url = url, Container = ext, VideoCodec = m.AudioOnly ? null : "h264", AudioCodec = "aac", Height = f.height, Bitrate = f.size, Size = f.size } );
m.Duration = MathF.Max( m.Duration, f.length );
}
return m;
}
/// <summary> "596.47" or "9:56" / "1:02:03". </summary>
static float Length( string s )
{
if ( string.IsNullOrEmpty( s ) ) return 0;
if ( !s.Contains( ':' ) ) return float.TryParse( s, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var f ) ? f : 0;
return s.Split( ':' ).Aggregate( 0f, ( t, part ) => t * 60 + (float.TryParse( part, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var v ) ? v : 0) );
}
}
/// <summary>
/// {artist}.bandcamp.com/track/{slug} (and /album/{slug}: its first track): the page's player data, mp3-128 stream.
/// </summary>
public sealed class BandcampExtractor : SiteExtractor
{
public override string Key => "bandcamp";
protected override string[] Hosts => new[] { "bandcamp.com" };
static readonly Regex TrAlbum = new( "data-tralbum=\"([^\"]+)\"", RegexOptions.Compiled );
protected override string ParseId( Uri url )
{
var p = url.AbsolutePath.Trim( '/' ).Split( '/', StringSplitOptions.RemoveEmptyEntries );
if ( p.Length < 2 || p[0] is not ("track" or "album") || url.Host.Equals( "bandcamp.com", StringComparison.OrdinalIgnoreCase ) ) return null;
return $"https://{url.Host.ToLowerInvariant()}/{p[0]}/{p[1]}";
}
protected override async Task<SiteMedia> FetchAsync( string id, CancellationToken ct )
{
using var r = await Http.RequestAsync( id, cancellationToken: ct );
if ( (int)r.StatusCode == 404 ) throw new ResolveException( "That Bandcamp page doesn't exist." );
if ( !r.IsSuccessStatusCode ) throw new ResolveException( $"Bandcamp refused the request ({(int)r.StatusCode})." );
var html = await r.Content.ReadAsStringAsync( ct );
var m = TrAlbum.Match( html );
if ( !m.Success ) throw new ResolveException( "Couldn't read that Bandcamp page." );
var data = JsonNode.Parse( System.Net.WebUtility.HtmlDecode( m.Groups[1].Value ) );
var track = data?["trackinfo"].Items().FirstOrDefault( t => t["file"]?["mp3-128"] is not null )
?? throw new ResolveException( "That Bandcamp release has no free stream (or needs buying)." );
var url = track["file"]["mp3-128"].Str();
return new SiteMedia
{
Title = $"{data["artist"].Str()} - {track["title"].Str()}",
Duration = track["duration"].Float(),
AudioOnly = true,
Files = { new MediaFormat { Id = "mp3-128", Url = url, Container = "mp3", AudioCodec = "mp3", Bitrate = 128000 } },
};
}
}
/// <summary>
/// Any .m3u8 link: a finished playlist plays as a seekable video, one that's still growing as a live stream.
/// </summary>
public sealed class HlsExtractor : IExtractor
{
public string Key => "hls";
static readonly string[] Extensions = { ".m3u8", ".m3u" };
public static bool IsPlaylist( Uri url )
=> (url.Scheme == Uri.UriSchemeHttp || url.Scheme == Uri.UriSchemeHttps) && Extensions.Contains( System.IO.Path.GetExtension( url.AbsolutePath ).ToLowerInvariant() );
public bool CanHandle( Uri url ) => IsPlaylist( url );
public async Task<MediaInfo> GetInfoAsync( Uri url, bool audioOnly, CancellationToken ct )
{
var (live, duration, _) = await Live.HlsReader.ProbeAsync( url, ct );
var name = Uri.UnescapeDataString( System.IO.Path.GetFileNameWithoutExtension( url.AbsolutePath ) );
return new MediaInfo
{
Extractor = Key,
Id = url.ToString(),
Title = name is "index" or "master" or "playlist" or "" ? url.Host : $"{url.Host} - {name}",
Duration = live ? 0 : (float)duration,
AudioOnly = audioOnly,
IsLive = live,
Merged = !live,
};
}
public async Task<StreamPlan> GetStreamsAsync( string id, StreamRequest request, CancellationToken ct )
{
var (live, _, _) = await Live.HlsReader.ProbeAsync( new Uri( id ), ct );
return new StreamPlan { Kind = live ? StreamKind.Live : StreamKind.Hls, DirectUrl = id };
}
}