Editor/HumanoidMocap/Inference/WindowsVideoDecoder.cs
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;

namespace HumanoidMocap.Inference;

public sealed record DecodedVideoFrame(byte[] Rgba,int Width,int Height,double Time);

/// <summary>Direct C# access to Windows Media Foundation. Sequential decoded samples carry
/// their actual presentation timestamps. No subprocess, Python, or frame-seeking approximation.
/// ABI slots and GUIDs are from Windows SDK 10.0.26100 mfobjects.h / mfreadwrite.h.</summary>
public sealed class WindowsVideoDecoder : IDisposable
{
    public const string ImplementationVersion="wmf-visible-oriented-v2";
    const int VideoStream=unchecked((int)0xfffffffc);
    IntPtr reader;
    bool started;
    int stride;
    VideoFrameLayout layout=null!;
    static readonly Guid Major=new("48eba18e-f8c9-4687-bf11-0a74c9f96a8f"),Subtype=new("f7e34c9a-42e8-4714-b74b-cb29d72c35e5"),
        Video=new("73646976-0000-0010-8000-00aa00389b71"),Rgb32=new("00000016-0000-0010-8000-00aa00389b71"),
        Processing=new("fb394f3d-ccf1-42ee-bbb3-f9b845d5681d"),Size=new("1652c33d-d6b2-4012-b834-72030849a37d"),
        Stride=new("644b4e48-1e02-4516-b0eb-c01ca9d49ac6"),Rotation=new("c380465d-2271-428c-9b83-ecea3b4a85c1"),
        MinimumAperture=new("d7388766-18fe-48c6-a177-ee894867c8c4"),GeometricAperture=new("66758743-7e5f-400d-980a-aa8596c85696");
    public WindowsVideoDecoder(string path)
    {
        if(!File.Exists(path))throw new FileNotFoundException("Video not found.",path);
        using var com=new ComScope();IntPtr attributes=IntPtr.Zero,type=IntPtr.Zero;
        try
        {
            Check(MFStartup(0x20070,0));started=true;Check(MFCreateAttributes(out attributes,1));
            Check(Method<SetInt>(attributes,21)(attributes,Processing,1));
            Check(MFCreateSourceReaderFromURL(Path.GetFullPath(path),attributes,out reader));
            Check(Method<SelectStream>(reader,4)(reader,unchecked((int)0xfffffffe),0));
            Check(Method<SelectStream>(reader,4)(reader,VideoStream,1));
            Check(MFCreateMediaType(out type));
            Check(Method<SetGuid>(type,24)(type,Major,Video));Check(Method<SetGuid>(type,24)(type,Subtype,Rgb32));
            Check(Method<SetType>(reader,7)(reader,VideoStream,IntPtr.Zero,type));ReadFormat();
        }
        // MF_E_TOPO_CODEC_NOT_FOUND / unsupported stream: Windows has no decoder for this video's codec.
        catch(COMException error) when(error.HResult is unchecked((int)0xC00D5212) or unchecked((int)0xC00D36C4) or unchecked((int)0xC00D36B4))
        {Dispose();throw new NotSupportedException(MissingCodecMessage(path),error);}
        catch{Dispose();throw;}
        finally{Release(ref type);Release(ref attributes);}
    }
    /// <summary>Names the codec when the container says so. iPhones record HEVC unless set to Most Compatible,
    /// and Windows decodes HEVC only with the HEVC Video Extensions installed.</summary>
    public const string UndecodableFramesMessage="Windows could not decode this video's frames. It may use an unusual encoding (4:4:4 or 10-bit H.264 from a screen recorder or editor) or be damaged. Re-export it as a standard H.264 MP4 (8-bit, 4:2:0) and upload it again.";
    public static string MissingCodecMessage(string path)
    {
        var codec="";
        try
        {
            using var stream=File.OpenRead(path);var window=(int)Math.Min(stream.Length,4*1024*1024);var bytes=new byte[window];
            foreach(var offset in new[]{0L,Math.Max(0,stream.Length-window)})
            {
                stream.Position=offset;var read=stream.Read(bytes,0,window);var text=System.Text.Encoding.Latin1.GetString(bytes,0,read);
                if(text.Contains("hvc1")||text.Contains("hev1")){codec="HEVC (H.265)";break;}
                if(text.Contains("av01")){codec="AV1";break;}
                if(text.Contains("vp09")){codec="VP9";break;}
            }
        }
        catch(IOException){}
        return codec.StartsWith("HEVC",StringComparison.Ordinal)
            ?"This video is HEVC (H.265), which this PC cannot decode. Install \"HEVC Video Extensions\" from the Microsoft Store and upload it again, or record in H.264: on iPhone choose Settings → Camera → Formats → Most Compatible."
            :$"Windows has no decoder for this video{(codec.Length>0?" ("+codec+")":"")}. Convert it to an H.264 MP4 and upload it again.";
    }
    void ReadFormat()
    {
        Check(Method<GetMediaType>(reader,6)(reader,VideoStream,out var type));
        try
        {
            Check(Method<GetLong>(type,8)(type,Size,out var dimensions));var width=(int)(dimensions>>32);var height=(int)(dimensions&uint.MaxValue);
            if(width<=0||height<=0||(long)width*height>33_177_600)throw new InvalidDataException("Video exceeds the 8K decode limit.");
            var hr=Method<GetInt>(type,7)(type,Stride,out stride);if(hr<0)stride=checked(width*4);
            if(Math.Abs((long)stride)<width*4L)throw new InvalidDataException("Invalid video stride.");
            var aperture=ReadAperture(type,MinimumAperture)??ReadAperture(type,GeometricAperture);
            var rotation=0;hr=Method<GetInt>(type,7)(type,Rotation,out var value);
            if(hr>=0)rotation=value;else if(hr!=unchecked((int)0xc00d36e6))Check(hr);
            layout=new(width,height,aperture?.X??0,aperture?.Y??0,aperture?.Width??width,aperture?.Height??height,rotation);
        }
        finally{Release(ref type);}
    }
    static (int X,int Y,int Width,int Height)? ReadAperture(IntPtr type,Guid key)
    {
        var bytes=new byte[16];var hr=Method<GetBlob>(type,15)(type,key,bytes,bytes.Length,out var size);
        if(hr==unchecked((int)0xc00d36e6))return null;Check(hr);
        if(size!=16)throw new InvalidDataException("Invalid video display aperture metadata.");
        if(BitConverter.ToUInt16(bytes,0)!=0||BitConverter.ToUInt16(bytes,4)!=0)
            throw new InvalidDataException("Fractional video aperture offsets are not supported. Export the video with square pixels and an integer crop.");
        return (BitConverter.ToInt16(bytes,2),BitConverter.ToInt16(bytes,6),BitConverter.ToInt32(bytes,8),BitConverter.ToInt32(bytes,12));
    }
    /// <summary>Seek to the preceding keyframe. Read forward to the requested presentation timestamp.</summary>
    public void Seek(double seconds)
    {
        if(!double.IsFinite(seconds)||seconds<0)throw new ArgumentOutOfRangeException(nameof(seconds));
        using var com=new ComScope();
        var position=new TimePosition{Type=20,Value=checked((long)(seconds*10_000_000))}; // VT_I8, 100 ns
        Check(Method<SetPosition>(reader,8)(reader,Guid.Empty,position));
    }
    public DecodedVideoFrame Read(CancellationToken token)
    {
        using var com=new ComScope();
        for(var attempts=0;attempts<1000;attempts++)
        {
            token.ThrowIfCancellationRequested();
            var result=Method<ReadSample>(reader,9)(reader,VideoStream,0,out _,out var flags,out var timestamp,out var sample);
            // The container opened but its frames cannot be decoded: 4:4:4 or 10-bit H.264 from screen
            // recorders and editors, or a damaged file. Windows reports only "unspecified error".
            if(result<0)throw new NotSupportedException(UndecodableFramesMessage,Marshal.GetExceptionForHR(result));
            IntPtr buffer=IntPtr.Zero;
            try
            {
                if((flags&1)!=0)throw new IOException("Windows video decoder reported an error.");
                if((flags&0x20)!=0)ReadFormat();
                if(sample==IntPtr.Zero){if((flags&2)!=0)return null;continue;}
                Check(Method<GetBuffer>(sample,41)(sample,out buffer));
                Check(Method<LockBuffer>(buffer,3)(buffer,out var data,out _,out var length));
                try
                {
                    return new(layout.CopyRgba(data,length,stride,token),layout.Width,layout.Height,timestamp/10_000_000d);
                }
                finally{Check(Method<NoArgs>(buffer,4)(buffer));}
            }
            finally{Release(ref buffer);Release(ref sample);}
        }
        throw new InvalidDataException("Video decoder produced too many empty samples.");
    }
    public void Dispose(){Release(ref reader);if(started){started=false;MFShutdown();}}
    static void Release(ref IntPtr pointer){if(pointer!=IntPtr.Zero){Marshal.Release(pointer);pointer=IntPtr.Zero;}}
    static T Method<T>(IntPtr pointer,int slot) where T:Delegate
    {if(pointer==IntPtr.Zero)throw new ObjectDisposedException(nameof(WindowsVideoDecoder));return Marshal.GetDelegateForFunctionPointer<T>(Marshal.ReadIntPtr(Marshal.ReadIntPtr(pointer),slot*IntPtr.Size));}
    static void Check(int hr){if(hr<0)Marshal.ThrowExceptionForHR(hr);}
    sealed class ComScope:IDisposable
    {
        readonly bool initialized;
        public ComScope(){var hr=CoInitializeEx(IntPtr.Zero,0);if(hr>=0)initialized=true;else if(hr!=unchecked((int)0x80010106))Check(hr);}
        public void Dispose(){if(initialized)CoUninitialize();}
    }
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int SetInt(IntPtr self,in Guid key,int value);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int SetGuid(IntPtr self,in Guid key,in Guid value);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int GetLong(IntPtr self,in Guid key,out ulong value);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int GetInt(IntPtr self,in Guid key,out int value);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int GetBlob(IntPtr self,in Guid key,[Out] byte[] bytes,int capacity,out int size);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int SelectStream(IntPtr self,int stream,int selected);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int SetType(IntPtr self,int stream,IntPtr reserved,IntPtr type);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int GetMediaType(IntPtr self,int stream,out IntPtr type);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int ReadSample(IntPtr self,int stream,int control,out int actual,out int flags,out long timestamp,out IntPtr sample);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int GetBuffer(IntPtr self,out IntPtr buffer);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int LockBuffer(IntPtr self,out IntPtr data,out int maximum,out int length);
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int NoArgs(IntPtr self);
    [StructLayout(LayoutKind.Explicit,Size=24)] struct TimePosition
    {
        [FieldOffset(0)] public ushort Type;
        [FieldOffset(8)] public long Value;
    }
    [UnmanagedFunctionPointer(CallingConvention.StdCall)] delegate int SetPosition(IntPtr self,in Guid format,in TimePosition position);
    [DllImport("mfplat.dll",ExactSpelling=true)] static extern int MFStartup(int version,int flags);
    [DllImport("mfplat.dll",ExactSpelling=true)] static extern int MFShutdown();
    [DllImport("mfplat.dll",ExactSpelling=true)] static extern int MFCreateAttributes(out IntPtr attributes,int count);
    [DllImport("mfplat.dll",ExactSpelling=true)] static extern int MFCreateMediaType(out IntPtr type);
    [DllImport("mfreadwrite.dll",ExactSpelling=true,CharSet=CharSet.Unicode)] static extern int MFCreateSourceReaderFromURL(string url,IntPtr attributes,out IntPtr reader);
    [DllImport("ole32.dll",ExactSpelling=true)] static extern int CoInitializeEx(IntPtr reserved,int flags);
    [DllImport("ole32.dll",ExactSpelling=true)] static extern void CoUninitialize();
}