wpf实现超低延迟的RTMP或RTSP播放_C#

技术背景

我们在做windows平台rtmp和rtsp播放模块对接的时候，有开发者需要在wpf下调用，如果要在wpf下使用，只需要参考c#的对接demo即可，唯一不同的是，视频流数据显示的话，要么通过控件模式，要么可以让rtmp、rtsp播放模块回调rgb数据上来，在wpf直接绘制即可。

技术实现

本文以大牛直播sdk的windows平台smartplayer为例，回调数据的模式，其他不再说明，只要处理好上来的数据就好：

播放之前，设置回调，选择nt_sp_e_video_frame_format_rgb32：

video_frame_call_back_ = new sp_sdkvideoframecallback(setvideoframecallback);

ntsmartplayersdk.nt_sp_setvideoframecallback(player_handle_, (int32)nt.ntsmartplayerdefine.nt_sp_e_video_frame_format.nt_sp_e_video_frame_format_rgb32, intptr.zero, video_frame_call_back_);

处理rgb数据回调：

       /*
        * nt_player_wrapper.cs
        * author: daniusdk.com
        */
        public void setvideoframecallback(intptr handle, intptr userdata, uint32 status, intptr frame)
        {
            if (frame == intptr.zero)
            {
                return;
            }

            //如需直接处理rgb数据，请参考以下流程
            nt_sp_videoframe video_frame = (nt_sp_videoframe)marshal.ptrtostructure(frame, typeof(nt_sp_videoframe));

            if (video_frame.format_ != (int32)nt.ntsmartplayerdefine.nt_sp_e_video_frame_format.nt_sp_e_video_frame_format_rgb32)
                return;

            nt_sp_videoframe pvideoframe = new nt_sp_videoframe();

            pvideoframe.format_ = video_frame.format_;
            pvideoframe.width_ = video_frame.width_;
            pvideoframe.height_ = video_frame.height_;

            pvideoframe.timestamp_ = video_frame.timestamp_;
            pvideoframe.stride0_ = video_frame.stride0_;
            pvideoframe.stride1_ = video_frame.stride1_;
            pvideoframe.stride2_ = video_frame.stride2_;
            pvideoframe.stride3_ = video_frame.stride3_;


            int32 argb_size = video_frame.stride0_ * video_frame.height_;

            pvideoframe.plane0_ = marshal.allochglobal(argb_size);
            copymemory(pvideoframe.plane0_, video_frame.plane0_, (uint32)argb_size);
        }

另外一种，可以用picturebox，在mainwindow.xaml 做以下设置：

        <windowsformshost horizontalalignment="left" height="338" margin="10,10,0,0" verticalalignment="top" width="480" background="black">
            <wf:picturebox x:name="realplaywnd"></wf:picturebox>
        </windowsformshost>

为了便于多实例集成参考，以播放2路为例（一路2560*1440，一路1920*1080）：

具体实现如下：

       /*
        * mainwindow.xaml.cs
        * author: daniusdk.com
        */
        public mainwindow()
        {
            initializecomponent();

            if (!initsdk())
                return;

            uidispatcher = dispatcher.currentdispatcher;

            player1_ = new nt_player_wrapper(realplaywnd, uidispatcher);
            player1_.eventgetplayereventmsg += new delgetplayereventmsg(getplayereventmsginfo);
            player1_.eventgetvideosize += new delgetvideosize(getvideosize);

            player2_ = new nt_player_wrapper(realplaywnd1, uidispatcher);
            player2_.eventgetplayereventmsg += new delgetplayereventmsg(getplayereventmsginfo);
            player2_.eventgetvideosize += new delgetvideosize(getvideosize);
        }

        private void getplayereventmsginfo(intptr handle, string msg)
        {
            this.dispatcher.invoke((action)delegate()
            {
                event_label.content = msg;
            });
        }

        private void getvideosize(intptr handle, string size)
        {
            this.dispatcher.invoke((action)delegate()
            {
                video_size.content = size;
            });
        }

        private bool initsdk()
        {
            if (!is_player_sdk_init_)
            {
                uint32 isplayerinited = nt.ntsmartplayersdk.nt_sp_init(0, intptr.zero);
                if (isplayerinited != 0)
                {
                    messagebox.show("调用nt_sp_init失败..");
                    return false;
                }

                is_player_sdk_init_ = true;
            }

            return true;
        }

        private void button_click_1(object sender, routedeventargs e)
        {
            if (!player1_.isplaying())
            {
                player1_.setbuffer(0);
                bool is_mute = true;

                if (!player1_.startplay("rtsp://admin:daniulive12345@192.168.0.120:554/h264/ch1/main/av_stream", false, is_mute))
                    return;

                btn_playback1.content = "停止播放";
            }
            else
            {
                player1_.stopplay();
                btn_playback1.content = "开始播放";
            }
        }

        private void button_click_2(object sender, routedeventargs e)
        {
            if (!player2_.isplaying())
            {
                player2_.setbuffer(0);
                bool is_mute = true;

                if (!player2_.startplay("rtsp://admin:admin123456@192.168.0.121:554/cam/realmonitor?channel=1&subtype=0", false, is_mute))
                    return;

                btn_playback2.content = "停止播放";
            }
            else
            {
                player2_.stopplay();
                btn_playback2.content = "开始播放";
            }
        }

        protected override void onclosing(system.componentmodel.canceleventargs e)
        {
            if (messagebox.show("确定要关闭窗口吗？", "确认", messageboxbutton.yesno) != messageboxresult.yes)
            {
                // 如果用户选择“否”，取消关闭
                e.cancel = true;
            }

            if (player1_.isplaying())
            {
                player1_.stopplay();
            }

            player1_.dispose();

            if (player2_.isplaying())
            {
                player2_.stopplay();
            }

            player2_.dispose();

            if (is_player_sdk_init_)
            {
                ntsmartplayersdk.nt_sp_uninit();
                is_player_sdk_init_ = false;
            }    

            base.onclosing(e);
        }

延迟依旧毫秒级，cpu占用如下，如果用硬解码，体验会更好：