Unity3d 如何通过TCP在Unity中播放来自GStreamer的流

Unity3d 如何通过TCP在Unity中播放来自GStreamer的流,unity3d,tcp,video-streaming,jpeg,gstreamer,Unity3d,Tcp,Video Streaming,Jpeg,Gstreamer,我正在进行一个项目,该项目需要使用GStreamer进行流式视频,然后我需要在Unity中显示视频 我现在正在尝试一种方法: 我使用下面的GStreamer命令将网络摄像头视图放到tcp上 gst-launch-1.0 v4l2src!视频转换!视频/x-raw,宽度=640,高度=480,帧速率=20/1!JPEG!多部件多工器!tcpserversink主机=127.0.0.1端口=5000 GStreamer可以通过以下方式查看: gst-launch-1.0 tcpclientsrc

我正在进行一个项目,该项目需要使用GStreamer进行流式视频,然后我需要在Unity中显示视频

我现在正在尝试一种方法:

  • 我使用下面的GStreamer命令将网络摄像头视图放到tcp上
  • gst-launch-1.0 v4l2src!视频转换!视频/x-raw,宽度=640,高度=480,帧速率=20/1!JPEG!多部件多工器!tcpserversink主机=127.0.0.1端口=5000
    GStreamer可以通过以下方式查看:

    gst-launch-1.0 tcpclientsrc  port=5000 host=127.0.0.1 !  multipartdemux   ! jpegdec  ! autovideosink
    
  • 在Unity中,我使用以下脚本获取流字节并对其进行解码
  • 但是,它不起作用。 我认为这是因为编码方法和解码方法不匹配,但我不知道如何解释GStreamer的字节序列。 在
    int imageSize=readImageByteSize(messagebytellength),,
    imageSize应使用如下所示的传入字节计算:
    frameBytesLength[]{196,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,}

    但是它看起来像这样: frameBytesLength[]{255、216、255、224、0、16、74、70、73、70、0、1、1、0、0、1、1、0、0、255、219、0、67、} 导致大小过大而变为负值,因此无法继续解码过程。 这就是我到目前为止所做的

    以下是统一代码的参考:

    p、 我使用的是Linux,所以我不能使用mray GStreamer Unity插件,因为它不支持Linux。
    谢谢您的时间。

    我以一种不太优雅的方式解决了这个问题。。。 我只是给接收到的每一帧分配一个固定的最大字节大小。
    这不是最好的方法,但对我来说很有效。

    Gstreamer是唯一的选择吗?您可以使用UMD(通用媒体播放器),它在流媒体或播放视频时更快速、更可靠。只是一个建议。没有恨:)嗨。我不能使用其他插件,虽然它会非常有用。。。因为它会占用太多的CPU,而我们的计算机无法处理它。无论如何谢谢你!
    public class _TextureReceiver : MonoBehaviour {
            public int port = 5000;
            public string IP = "127.0.0.1";
            TcpClient client;
    
            [HideInInspector]
            public Texture2D texture;
    
            private bool stop = false;
    
            [Header("Must be the same in sender and receiver")]
            public int messageByteLength = 24;
    
            // Use this for initialization
            void Start() {
                Application.runInBackground = true;
    
                client = new TcpClient();
    
                //Connect to server from another Thread
                Loom.RunAsync(() => {                
                    // if on desktop
                    // client.Connect(IPAddress.Loopback, port);
                    client.Connect(IPAddress.Parse(IP), port);
    
                    imageReceiver();
                });
            }
            void imageReceiver() {
                //While loop in another Thread is fine so we don't block main Unity Thread
                Loom.RunAsync(() => {
                    while (!stop) {
                        //Read Image Count
                        int imageSize = readImageByteSize(messageByteLength);
    
                        //Read Image Bytes and Display it
                        readFrameByteArray(imageSize);
                        //readFrameByteArray(12288000);
                    }
                });
            }
    
            //Converts the byte array to the data size and returns the result
            int frameByteArrayToByteLength(byte[] frameBytesLength) {
                /*if (BitConverter.IsLittleEndian)
                 {
                     Array.Reverse(frameBytesLength);
                     UnityEngine.Debug.Log("- BitConverter.IsLittleEndian");
                 }*/
                var sb = new StringBuilder("frameBytesLength[] { ");
                foreach (var b in frameBytesLength)
                {
                    sb.Append(b + ", ");
                }
                sb.Append("}");
                UnityEngine.Debug.Log(sb.ToString());
                //int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
                int byteLength = frameBytesLength[0] + frameBytesLength[1] * 256;
                return byteLength;
            }
    
            private int readImageByteSize(int size) {
                UnityEngine.Debug.Log("- image byte size: " + size);
                bool disconnected = false;
    
                NetworkStream serverStream = client.GetStream();
                UnityEngine.Debug.Log("- serverStream: " + serverStream);
                byte[] imageBytesCount = new byte[size];
                var total = 0;
                do {
                    var read = serverStream.Read(imageBytesCount, total, size - total);
                    UnityEngine.Debug.LogFormat("Client recieved "+read+" bytes");
                    if (read == 0)
                    {
                    disconnected = true;
                    break;
                    }
                    total += read;
                    UnityEngine.Debug.Log("- image byte read: " + read);
                    UnityEngine.Debug.Log("- image byte total: " + total);
                } while (total != size);
                UnityEngine.Debug.Log("- break While");
                int byteLength;
    
                if (disconnected) {
                    UnityEngine.Debug.Log("disconnected");
                    byteLength = -1;
                } else {
                    byteLength = frameByteArrayToByteLength(imageBytesCount);
                }
    
                return byteLength;
            }
    
            private void readFrameByteArray(int size) {
                bool disconnected = false;
                UnityEngine.Debug.Log("- image size: " + size);
                NetworkStream serverStream = client.GetStream();
                byte[] imageBytes = new byte[size];
                var total = 0;
                //do{
                    var read = serverStream.Read(imageBytes, total, size - total);
                    if (read == 0)
                    {
                        disconnected = true;
                        //break;
                    }
                    total += read;
                    UnityEngine.Debug.Log("- read: " + read);
                    UnityEngine.Debug.Log("- total: " + total);
                //} while (total != size);
                byte[] imageBytes2 = new byte[read];
                imageBytes.CopyTo(imageBytes2, 0);
    
                UnityEngine.Debug.Log("break while");
    
                var sb = new StringBuilder("imageBytes[] { ");
                foreach (var b in imageBytes2)
                {
                    sb.Append(b + ", ");
                }
                sb.Append("}");
                UnityEngine.Debug.Log(sb.ToString());
    
                bool readyToReadAgain = false;
    
                //Display Image
                if (!disconnected) {
                    //Display Image on the main Thread
                    Loom.QueueOnMainThread(() => {
                        loadReceivedImage(imageBytes2);
                        readyToReadAgain = true;
                    });
                }
    
                //Wait until old Image is displayed
                while (!readyToReadAgain) {
                    System.Threading.Thread.Sleep(1);
                }
            }
    
    
            void loadReceivedImage(byte[] receivedImageBytes) {
                if(texture) texture.LoadImage(receivedImageBytes);
            }
    
            public void SetTargetTexture (Texture2D t) {
                texture = t;
            }
    
            void OnApplicationQuit() {
                stop = true;
    
                if (client != null) {
                client.Close();
                }
            }
        }