C# gstreamer，在winforms（和WPF）中呈现rtspsrc

我试图编写的应用程序基于我网络上的视频服务器获取视频流，并将其显示在winforms窗口中（稍后我希望在WPF中承载相同类型的控件）。我正在使用gstreamer sharp，因为我的应用程序基于c#net

我成功地使videotestsrc基于中的代码示例工作，并能够使用VideoOverlayaAdapter和一组winForms面板创建testvideosrc的多个实例，并根据需要在窗口中显示

当我开始让rtspsrc做同样的事情时，我很自然地遇到了一些我试图克服的障碍，下面是我的类的代码

我认为我需要将rtspsrc的新pad链接到下一个元素（在本例中为rtph264depay），而不是在初始化代码中链接rtspsrc，这就是我遇到麻烦的地方

PadAdded事件似乎有时会在启动程序的几秒钟内触发，有时根本不会触发？该服务器与gstreamer sharp版本的基本教程（第1部分）配合良好，并且具有良好的延迟（很容易小于300ms，但一旦我的应用程序正常工作，我需要进行一次玻璃对玻璃的测试）

此外，一旦PadAdded事件最终触发，在尝试将新焊盘链接到rtph264depay接收器焊盘时，我将获得NOFORMAT状态

我还注意到，我似乎没有收到prepare window handle bus sync消息，我将像gstVideoOverlay示例中那样设置视频覆盖适配器（因此，即使pad链接成功，我也不会得到所需窗口句柄的输出）

我没有发现这个特殊的问题（rtspsrc焊盘没有链接到rtph264depay水槽焊盘），因为类似的问题似乎是关于将其他元素链接在一起

根据调试消息，初始化代码中其余元素的初始链接成功

最终目标是将这些帧放入OpenCV/Emgu，并进行一些分析和基本的叠加工作

在此方面的任何帮助都将不胜感激

非常感谢

/// <summary>
/// class to create a gstreamer pipeline based on an rtsp stream at the provided URL
/// </summary>
class gstPipeline2
{
    // elements for the pipeline
    private Element rtspsrc, rtph264depay, decoder, videoConv, videoSink;
    private System.Threading.Thread mainGLibThread;
    private GLib.MainLoop mainLoop;

    // the window handle (passed in)
    private IntPtr windowHandle;
    // our pipeline
    private Pipeline currentPipeline = null;

    /// <summary>
    /// Create a new gstreamer pipeline rendering the stream at URL into the provided window handle 
    /// </summary>
    /// <param name="WindowHandle">The handle of the window to render to </param>
    /// <param name="Url">The url of the video stream</param>
    public gstPipeline2(IntPtr WindowHandle, string Url)
    {
        windowHandle = WindowHandle;    // get the handle and save it locally

        // initialise the gstreamer library and associated threads (for diagnostics)
        Gst.Application.Init(); 
        mainLoop = new GLib.MainLoop();
        mainGLibThread = new System.Threading.Thread(mainLoop.Run);
        mainGLibThread.Start();

        // create each element now for the pipeline
        // starting with the rtspsrc
        rtspsrc = ElementFactory.Make("rtspsrc", "udpsrc0");  // create an rtsp source
        rtspsrc["location"] = Url;   // and set its location (the source of the data)
        rtph264depay = ElementFactory.Make("rtph264depay", "rtph264depay0");    
        decoder = ElementFactory.Make("avdec_h264", "decoder0");    
        videoConv = ElementFactory.Make("videoconvert", "videoconvert0");   
        videoSink = ElementFactory.Make("autovideosink", "sink0");  // and finally the sink to render the video (redirected to the required window handle below in Bus_SyncMessage() ) 

        // create our pipeline which links all the elements together into a valid data flow
        currentPipeline = new Pipeline("pipeline");
        currentPipeline.Add(rtspsrc, rtph264depay, decoder, videoConv, videoSink); // add the required elements into it

        // link the various bits together in the correct order
        if(!rtph264depay.Link(decoder))
            System.Diagnostics.Debug.WriteLine("rtph264depay could not be linked to decoder (bad)");
        else
            System.Diagnostics.Debug.WriteLine("rtph264depay linked to decoder (good)");

        if (!decoder.Link(videoConv))
            System.Diagnostics.Debug.WriteLine("decoder could not be linked to videoconvert (bad)");
        else
            System.Diagnostics.Debug.WriteLine("decoder linked to videoconvert (good)");

        if (!videoConv.Link(videoSink))
            System.Diagnostics.Debug.WriteLine("videoconvert could not be linked to autovideosink (bad)");
        else
            System.Diagnostics.Debug.WriteLine("videoconvert linked to autovideosink (good)");

        rtspsrc.PadAdded += Rtspsrc_PadAdded; // subscribe to the PadAdded event so we can link new pads (sources of data?) to the depayloader when they arrive

        // subscribe to the messaging system of the bus and pipeline so we can minotr status as we go
        Bus bus = currentPipeline.Bus;
        bus.AddSignalWatch();
        bus.Message += Bus_Message;

        bus.EnableSyncMessageEmission();
        bus.SyncMessage += Bus_SyncMessage;

        // finally set the state of the pipeline running so we can get data
        var setStateReturn = currentPipeline.SetState(State.Null);
        System.Diagnostics.Debug.WriteLine("SetStateNULL returned: " + setStateReturn.ToString());
        setStateReturn = currentPipeline.SetState(State.Ready);
        System.Diagnostics.Debug.WriteLine("SetStateReady returned: " + setStateReturn.ToString());
        setStateReturn = currentPipeline.SetState(State.Playing);
        System.Diagnostics.Debug.WriteLine("SetStatePlaying returned: " + setStateReturn.ToString());
    }

    private void Rtspsrc_PadAdded(object o, PadAddedArgs args)
    {
        System.Diagnostics.Debug.WriteLine("Rtspsrc_PadAdded: called with new pad named: " + args.NewPad.Name);

        // a pad has been added to the source so we need to link it to the rest of the pipeline to ultimately display it onscreen
        Pad sinkPad = rtph264depay.GetStaticPad("sink");   // get the sink pad for the one we have recieved  so we can link to the depayloader element
        System.Diagnostics.Debug.WriteLine("Rtspsrc_PadAdded: rtps264depay sink pad returned: " + sinkPad.Name);

        PadLinkReturn ret = args.NewPad.Link(sinkPad);
        System.Diagnostics.Debug.WriteLine("Rtspsrc_PadAdded: link attempt returned: " + ret.ToString());
    }

    public void killProcess()
    {
        mainLoop.Quit();
    }

    private void Bus_SyncMessage(object o, SyncMessageArgs args)
    {
        if (Gst.Video.Global.IsVideoOverlayPrepareWindowHandleMessage(args.Message))
        {
            System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Message prepare window handle received by: " + args.Message.Src.Name + " " + args.Message.Src.GetType().ToString());

            if (args.Message.Src != null)
            {
                // these checks were in the testvideosrc example and failed, args.Message.Src is always Gst.Element???
                if (args.Message.Src is Gst.Video.VideoSink)
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is VideoSink");
                else
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is NOT VideoSink");

                if (args.Message.Src is Gst.Bin)
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is Bin");
                else
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is NOT Bin");

                try
                {
                    args.Message.Src["force-aspect-ratio"] = true;
                }
                catch (PropertyNotFoundException) { }

                try
                {
                    Gst.Video.VideoOverlayAdapter adapter = new VideoOverlayAdapter(args.Message.Src.Handle);
                    adapter.WindowHandle = windowHandle;
                    adapter.HandleEvents(true);
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Handle passed to adapter: " + windowHandle.ToString());
                }
                catch (Exception ex) { System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Exception Thrown (overlay stage): " + ex.Message); }
            }
        }
        else
        {
            string info;
            IntPtr prt;
            args.Message.ParseInfo(out prt, out info);
            System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: " + args.Message.Type.ToString() + " - " + info);
        }
    }

    private void Bus_Message(object o, MessageArgs args)
    {
        var msg = args.Message;
        //System.Diagnostics.Debug.WriteLine("HandleMessage received msg of type: {0}", msg.Type);
        switch (msg.Type)
        {
            case MessageType.Error:
                //
                GLib.GException err;
                string debug;
                System.Diagnostics.Debug.WriteLine("Bus_Message: Error received: " + msg.ToString());
                break;
            case MessageType.StreamStatus:
                Gst.StreamStatusType status;
                Element theOwner;
                msg.ParseStreamStatus(out status, out theOwner);
                System.Diagnostics.Debug.WriteLine("Bus_Message: Case SteamingStatus: status is: " + status + " ; Owner is: " + theOwner.Name);
                break;
            case MessageType.StateChanged:
                State oldState, newState, pendingState;
                msg.ParseStateChanged(out oldState, out newState, out pendingState);
                if (newState == State.Paused)
                    args.RetVal = false;
                System.Diagnostics.Debug.WriteLine("Bus_Message: Pipeline state changed from {0} to {1}: ; Pending: {2}", Element.StateGetName(oldState), Element.StateGetName(newState), Element.StateGetName(pendingState));
                break;
            case MessageType.Element:
                System.Diagnostics.Debug.WriteLine("Bus_Message: Element message: {0}", args.Message.ToString());
                break;
            default:
                System.Diagnostics.Debug.WriteLine("Bus_Message: HandleMessage received msg of type: {0}", msg.Type);
                break;
        }
        args.RetVal = true;
    }
}

//
///类以基于所提供URL处的rtsp流创建gstreamer管道
/// 
gstPipeline2类
{
//管道的要素
专用元素rtspsrc、rtph264depay、解码器、videoConv、videoSink；
private System.Threading.Thread mainGLibThread；
私有GLib.MainLoop MainLoop；
//窗口句柄（传入）
私有IntPtr窗口句柄；
//我们的管道
私有管道currentPipeline=null；
/// 
///创建一个新的gstreamer管道，将URL处的流呈现到提供的窗口句柄中
/// 
///要渲染到的窗口的句柄
///视频流的url
公共gstPipeline2（IntPtr WindowHandle，字符串Url）
{
windowHandle=windowHandle；//获取句柄并将其保存到本地
//初始化gstreamer库和相关线程（用于诊断）
Gst.Application.Init（）；
mainLoop=新GLib.mainLoop（）；
mainGLibThread=new System.Threading.Thread（mainLoop.Run）；
mainGLibThread.Start（）；
//现在为管道创建每个元素
//从rtspsrc开始
rtspsrc=ElementFactory.Make（“rtspsrc”、“udpsrc0”）；//创建一个rtsp源
rtspsrc[“location”]=Url；//并设置其位置（数据源）
rtph264depay=ElementFactory.Make（“rtph264depay”、“rtph264depay0”）；
解码器=ElementFactory.Make（“avdec_h264”，“解码器0”）；
videoConv=ElementFactory.Make（“videoconvert”、“videoconvert0”）；
videoSink=ElementFactory.Make（“autovideosink”，“sink0”）；//最后是渲染视频的接收器（重定向到Bus_SyncMessage（）中下面所需的窗口句柄）
//创建将所有元素链接到一个有效数据流中的管道
currentPipeline=新管道（“管道”）；
currentPipeline.Add（rtspsrc、rtph264depay、解码器、videoConv、videoSink）；//向其中添加所需的元素
//按正确顺序将各个位链接在一起
如果（！rtph264depay.Link（解码器））
System.Diagnostics.Debug.WriteLine（“rtph264depay无法链接到解码器（坏）”；
其他的
System.Diagnostics.Debug.WriteLine（“rtph264depay链接到解码器（良好）”；
if（！解码器链接（videoConv））
System.Diagnostics.Debug.WriteLine（“解码器无法链接到videoconvert（坏）”）；
其他的
System.Diagnostics.Debug.WriteLine（“解码器链接到videoconvert（良好）”）；
如果（！videoConv.Link（videoSink））
System.Diagnostics.Debug.WriteLine（“videoconvert无法链接到autovideosink（错误）”；
其他的
System.Diagnostics.Debug.WriteLine（“连接到autovideosink的视频转换（良好）”；
rtspsrc.paddadded+=rtspsrc_paddadded；//订阅paddadded事件，以便我们可以在新pad（数据源？）到达时将其链接到depayloader
//订阅总线和管道的消息传递系统，这样我们就可以在运行时查看minotr状态
Bus Bus=currentPipeline.Bus；
bus.AddSignalWatch（）；
总线消息+=总线消息；
bus.EnableSyncMessageEmission（）；
bus.SyncMessage+=总线\同步消息；
//最后设置管道运行的状态，以便获取数据
var setStateReturn=currentPipeline.SetState（State.Null）；
System.Diagnostics.Debug.WriteLine（“SetStateNULL返回：”+setStateReturn.ToString（））；
setStateReturn=currentPipeline.SetState（State.Ready）；
System.Diagnostics.Debug.WriteLine（“SetStateReady返回：”+setStateReturn.ToString（））；
setStateReturn=currentPipeline.SetState（State.Playing）；
System.Diagnostics.Debug.WriteLine（“SetStatePlaying返回：+setStateReturn.ToString（））；
}
私有void rtspsrcu paddadded（对象o，paddaddar
/// <summary>
/// class to create a gstreamer pipeline based on an rtsp stream at the provided URL
/// </summary>
class gstPipeline2
{
    // elements for the pipeline
    private Element uriDecodeBin, videoSink;
    private System.Threading.Thread mainGLibThread;
    private GLib.MainLoop mainLoop;

    // the window handle (passed in)
    private IntPtr windowHandle;
    // our pipeline
    private Pipeline currentPipeline = null;

    /// <summary>
    /// Create a new gstreamer pipeline rendering the stream at URL into the provided window handle 
    /// </summary>
    /// <param name="WindowHandle">The handle of the window to render to </param>
    /// <param name="Url">The url of the video stream</param>
    public gstPipeline2(string Url, IntPtr WindowHandle)
    {
        windowHandle = WindowHandle;    // get the handle and save it locally

        // initialise the gstreamer library and associated threads (for diagnostics)
        Gst.Application.Init();

        mainLoop = new GLib.MainLoop();
        mainGLibThread = new System.Threading.Thread(mainLoop.Run);
        mainGLibThread.Start();

        // create each element now for the pipeline
        uriDecodeBin = ElementFactory.Make("uridecodebin", "uriDecodeBin0");  // create an uridecodebin (which handles most of the work for us!!)
        uriDecodeBin["uri"] = Url;   // and set its location (the source of the data)
        videoSink = ElementFactory.Make("autovideosink", "sink0");  // and finally the sink to render the video (redirected to the required window handle below in Bus_SyncMessage() ) 

        // create our pipeline which links all the elements together into a valid data flow
        currentPipeline = new Pipeline("pipeline");
        currentPipeline.Add(uriDecodeBin, videoSink); // add the required elements into it

        uriDecodeBin.PadAdded += uriDecodeBin_PadAdded; // subscribe to the PadAdded event so we can link new pads (sources of data?) to the depayloader when they arrive
        uriDecodeBin.Connect("source-setup", SourceSetup);  // subscribe to the "source-setup" signal, not quite done in the usual C# eventing way but treat it as essentially the same

        // subscribe to the messaging system of the bus and pipeline so we can monitor status as we go
        Bus bus = currentPipeline.Bus;
        bus.AddSignalWatch();
        bus.Message += Bus_Message;

        bus.EnableSyncMessageEmission();
        bus.SyncMessage += Bus_SyncMessage;

        // finally set the state of the pipeline running so we can get data
        var setStateReturn = currentPipeline.SetState(State.Null);
        System.Diagnostics.Debug.WriteLine("SetStateNULL returned: " + setStateReturn.ToString());
        setStateReturn = currentPipeline.SetState(State.Ready);
        System.Diagnostics.Debug.WriteLine("SetStateReady returned: " + setStateReturn.ToString());
        setStateReturn = currentPipeline.SetState(State.Playing);
        System.Diagnostics.Debug.WriteLine("SetStatePlaying returned: " + setStateReturn.ToString());
    }

    private void uriDecodeBin_PadAdded(object o, PadAddedArgs args)
    {
        System.Diagnostics.Debug.WriteLine("uriDecodeBin_PadAdded: called with new pad named: " + args.NewPad.Name);

        // a pad has been added to the source so we need to link it to the rest of the pipeline to ultimately display it onscreen
        Pad sinkPad = videoSink.GetStaticPad("sink");   // get the pad for the one we have recieved  so we can link to the depayloader element
        System.Diagnostics.Debug.WriteLine("uriDecodeBin_PadAdded: queue pad returned: " + sinkPad.Name);

        PadLinkReturn ret = args.NewPad.Link(sinkPad);

        System.Diagnostics.Debug.WriteLine("uriDecodeBin_PadAdded: link attempt returned: " + ret.ToString());
    }

    void SourceSetup(object sender, GLib.SignalArgs args)
    {
        // we need to delve into the source portion of the uridecodebin to modify the "latency" property, need to add some validation here to ensure this is an rtspsrc
        var source = (Element)args.Args[0];
        System.Diagnostics.Debug.WriteLine("SourceSetup: source is named: " + source.Name + ", and is of type: " + source.NativeType.ToString());
        source["latency"] = 0;  // this COULD throw an exception if the source is not rtspsrc or similar with a "latency" property
    }

    public void killProcess()
    {
        mainLoop.Quit();
    }

    private void Bus_SyncMessage(object o, SyncMessageArgs args)
    {
        if (Gst.Video.Global.IsVideoOverlayPrepareWindowHandleMessage(args.Message))
        {
            System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Message prepare window handle received by: " + args.Message.Src.Name + " " + args.Message.Src.GetType().ToString());

            if (args.Message.Src != null)
            {
                // these checks were in the testvideosrc example and failed, args.Message.Src is always Gst.Element???
                if (args.Message.Src is Gst.Video.VideoSink)
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is VideoSink");
                else
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is NOT VideoSink");

                if (args.Message.Src is Gst.Bin)
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is Bin");
                else
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: source is NOT Bin");

                try
                {
                    args.Message.Src["force-aspect-ratio"] = true;
                }
                catch (PropertyNotFoundException) { }

                try
                {
                    Gst.Video.VideoOverlayAdapter adapter = new VideoOverlayAdapter(args.Message.Src.Handle);
                    adapter.WindowHandle = windowHandle;
                    adapter.HandleEvents(true);
                    System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Handle passed to adapter: " + windowHandle.ToString());
                }
                catch (Exception ex) { System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: Exception Thrown (overlay stage): " + ex.Message); }
            }
        }
        else
        {
            string info;
            IntPtr prt;
            args.Message.ParseInfo(out prt, out info);
            System.Diagnostics.Debug.WriteLine("Bus_SyncMessage: " + args.Message.Type.ToString() + " - " + info);
        }
    }

    private void Bus_Message(object o, MessageArgs args)
    {
        var msg = args.Message;
        //System.Diagnostics.Debug.WriteLine("HandleMessage received msg of type: {0}", msg.Type);
        switch (msg.Type)
        {
            case MessageType.Error:
                //
                GLib.GException err;
                string debug;
                System.Diagnostics.Debug.WriteLine("Bus_Message: Error received: " + msg.ToString());
                break;
            case MessageType.StreamStatus:
                Gst.StreamStatusType status;
                Element theOwner;
                msg.ParseStreamStatus(out status, out theOwner);
                System.Diagnostics.Debug.WriteLine("Bus_Message: Case SteamingStatus: status is: " + status + " ; Owner is: " + theOwner.Name);
                break;
            case MessageType.StateChanged:
                State oldState, newState, pendingState;
                msg.ParseStateChanged(out oldState, out newState, out pendingState);
                if (newState == State.Paused)
                    args.RetVal = false;
                System.Diagnostics.Debug.WriteLine("Bus_Message: Pipeline state changed from {0} to {1}: ; Pending: {2}", Element.StateGetName(oldState), Element.StateGetName(newState), Element.StateGetName(pendingState));
                break;
            case MessageType.Element:
                System.Diagnostics.Debug.WriteLine("Bus_Message: Element message: {0}", args.Message.ToString());
                break;
            default:
                System.Diagnostics.Debug.WriteLine("Bus_Message: HandleMessage received msg of type: {0}", msg.Type);
                break;
        }
        args.RetVal = true;
    }
}