如何通过TCP在Unity中播放GStreamer视频流及故障排查

你遇到的核心问题是GStreamer推流的multipartmux输出格式和Unity脚本的读取逻辑不匹配：

• 你用multipartmux生成的是多部分MIME格式的流，结构是--边界字符串 + 头部信息（Content-Length等） + JPEG数据，并非先固定发送24字节的长度值。
• 而你的Unity脚本硬要先读24字节当长度，自然会读到JPEG文件的起始头（0xFFD8就是JPEG的SOI标记），导致长度计算完全错误。

下面给你两种适配Linux环境的解决方案，任选其一即可：

方案一：修改GStreamer推流逻辑，发送「长度+JPEG」自定义格式 #

这种方式适配你现有Unity脚本的核心逻辑，只需要用Python写一个简单的中转推流脚本（Linux下Python环境默认自带，无需额外依赖），替代原来的gst-launch命令：

1. Python推流脚本（发送长度+JPEG数据） #

import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject
import socket

# 初始化GStreamer
Gst.init(None)

# 启动TCP服务器等待Unity连接
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(('127.0.0.1', 5000))
server_socket.listen(1)
print("等待Unity客户端连接...")
client_conn, addr = server_socket.accept()
print(f"已连接到 {addr}")

# 构建GStreamer管道，用appsink获取JPEG帧
pipeline = Gst.parse_launch(
    "v4l2src ! videoconvert ! video/x-raw,width=640,height=480,framerate=20/1 ! jpegenc ! appsink name=sink"
)
appsink = pipeline.get_by_name("sink")
appsink.set_property("emit-signals", True)

# 每收到一帧就发送长度+数据
def on_new_sample(sink):
    sample = sink.emit("pull-sample")
    buf = sample.get_buffer()
    success, map_info = buf.map(Gst.MapFlags.READ)
    if success:
        # 把帧长度转成4字节大端格式（Unity端统一解析）
        data_len = map_info.size
        len_bytes = data_len.to_bytes(4, byteorder='big')
        # 先发长度，再发JPEG数据
        client_conn.sendall(len_bytes)
        client_conn.sendall(map_info.data)
        buf.unmap(map_info)
    return Gst.FlowReturn.OK

appsink.connect("new-sample", on_new_sample)

# 启动管道并保持运行
pipeline.set_state(Gst.State.PLAYING)
loop = GObject.MainLoop()
try:
    loop.run()
except KeyboardInterrupt:
    pass

# 资源清理
pipeline.set_state(Gst.State.NULL)
client_conn.close()
server_socket.close()

2. Unity脚本修改（适配新的长度解析逻辑） #

只需要修改两个关键部分：

• 将messageByteLength改为4（对应Python发送的4字节长度）
• 修复frameByteArrayToByteLength函数，正确解析大端格式的长度：

int frameByteArrayToByteLength(byte[] frameBytesLength) {
    // Python发送的是大端字节序，Unity默认小端的话需要反转
    if (BitConverter.IsLittleEndian) {
        Array.Reverse(frameBytesLength);
    }
    return BitConverter.ToInt32(frameBytesLength, 0);
}

• 修复readFrameByteArray的循环逻辑（原来的代码只读取一次，可能读不全数据）：

private void readFrameByteArray(int size) {
    bool disconnected = false;
    UnityEngine.Debug.Log("- image size: " + size);
    NetworkStream serverStream = client.GetStream();
    byte[] imageBytes = new byte[size];
    var total = 0;
    
    // 循环读取直到拿到完整的JPEG数据
    do {
        var read = serverStream.Read(imageBytes, total, size - total);
        if (read == 0) {
            disconnected = true;
            break;
        }
        total += read;
    } while (total != size);

    if (disconnected) {
        UnityEngine.Debug.Log("disconnected");
        return;
    }

    // 在主线程更新纹理
    Loom.QueueOnMainThread(() => {
        loadReceivedImage(imageBytes);
    });
}

方案二：Unity脚本直接解析multipart MIME格式 #

这种方式不需要修改GStreamer推流命令，直接在Unity里解析multipartmux的标准输出格式：

修改后的Unity核心逻辑 #

public class _TextureReceiver : MonoBehaviour {
    public int port = 5000;
    public string IP = "127.0.0.1";
    TcpClient client;
    [HideInInspector] public Texture2D texture;
    private bool stop = false;

    // multipartmux的默认边界字符串
    private string boundary = "--GstMultipartBoundary";
    private byte[] boundaryBytes;
    private byte[] crlfBytes = System.Text.Encoding.ASCII.GetBytes("\r\n");

    void Start() {
        Application.runInBackground = true;
        client = new TcpClient();
        boundaryBytes = System.Text.Encoding.ASCII.GetBytes(boundary);
        // 初始化Texture2D（根据推流分辨率设置）
        texture = new Texture2D(640, 480, TextureFormat.RGB24, false);

        Loom.RunAsync(() => {
            client.Connect(IPAddress.Parse(IP), port);
            imageReceiver();
        });
    }

    void imageReceiver() {
        Loom.RunAsync(() => {
            NetworkStream serverStream = client.GetStream();
            while (!stop) {
                // 1. 读取边界字符串
                if (!ReadUntilMatch(serverStream, boundaryBytes)) {
                    UnityEngine.Debug.Log("读取边界时断开连接");
                    break;
                }
                // 跳过边界后的换行
                ReadExact(serverStream, crlfBytes);

                // 2. 解析头部信息，获取Content-Length
                Dictionary<string, string> headers = new Dictionary<string, string>();
                while (true) {
                    string line = ReadLine(serverStream);
                    if (string.IsNullOrEmpty(line)) break; // 空行表示头部结束
                    int colonIndex = line.IndexOf(':');
                    if (colonIndex > 0) {
                        string key = line.Substring(0, colonIndex).Trim();
                        string value = line.Substring(colonIndex+1).Trim();
                        headers[key] = value;
                    }
                }

                // 3. 获取JPEG数据长度
                if (!headers.TryGetValue("Content-Length", out string lenStr) || !int.TryParse(lenStr, out int imageSize)) {
                    UnityEngine.Debug.Log("无法获取Content-Length");
                    continue;
                }

                // 4. 读取完整的JPEG数据
                byte[] imageBytes = new byte[imageSize];
                int totalRead = 0;
                bool disconnected = false;
                while (totalRead < imageSize) {
                    int read = serverStream.Read(imageBytes, totalRead, imageSize - totalRead);
                    if (read == 0) {
                        disconnected = true;
                        break;
                    }
                    totalRead += read;
                }

                if (disconnected) {
                    UnityEngine.Debug.Log("读取图片数据时断开连接");
                    break;
                }

                // 5. 在主线程更新纹理
                Loom.QueueOnMainThread(() => {
                    loadReceivedImage(imageBytes);
                });
            }
        });
    }

    // 辅助函数：读取直到匹配目标字节数组
    private bool ReadUntilMatch(NetworkStream stream, byte[] target) {
        int targetIndex = 0;
        byte[] buffer = new byte[1];
        while (targetIndex < target.Length) {
            int read = stream.Read(buffer, 0, 1);
            if (read == 0) return false;
            if (buffer[0] == target[targetIndex]) {
                targetIndex++;
            } else {
                targetIndex = 0; // 不匹配则重置索引
            }
        }
        return true;
    }

    // 辅助函数：读取精确长度的字节
    private bool ReadExact(NetworkStream stream, byte[] buffer) {
        int totalRead = 0;
        while (totalRead < buffer.Length) {
            int read = stream.Read(buffer, totalRead, buffer.Length - totalRead);
            if (read == 0) return false;
            totalRead += read;
        }
        return true;
    }

    // 辅助函数：读取一行文本（直到CRLF）
    private string ReadLine(NetworkStream stream) {
        StringBuilder sb = new StringBuilder();
        byte[] buffer = new byte[1];
        while (true) {
            int read = stream.Read(buffer, 0, 1);
            if (read == 0) return sb.ToString();
            if (buffer[0] == '\r') {
                stream.Read(buffer, 0, 1); // 跳过换行符\n
                break;
            } else if (buffer[0] != '\n') {
                sb.Append((char)buffer[0]);
            }
        }
        return sb.ToString();
    }

    void loadReceivedImage(byte[] receivedImageBytes) {
        if(texture) texture.LoadImage(receivedImageBytes);
    }

    public void SetTargetTexture (Texture2D t) {
        texture = t;
    }

    void OnApplicationQuit() {
        stop = true;
        if (client != null) {
            client.Close();
        }
    }
}

额外注意事项 #

1. Texture初始化：确保Unity的Texture2D是按照推流的分辨率（640x480）初始化的，否则LoadImage可能无法正确显示。
2. 权限问题：Linux下要确保Unity进程有摄像头访问权限（可以用sudo chmod 666 /dev/video0临时授权）。
3. 连接顺序：先启动GStreamer推流（或Python脚本），再启动Unity，避免连接失败。

内容的提问来源于stack exchange，提问作者Yun-Chien Lo