如何将OpenCV的Mat转换为Unity的Texture2D？

Great question! Let's walk through how to properly convert your OpenCV cv::Mat to a Unity Texture2D—right now your GetRawImage() only returns raw byte data, which isn't enough on its own. We need a few extra pieces of info from the C++ side to make this work reliably.

Step 1: Expand Your C++ Exports #

The raw byte pointer doesn't tell Unity anything about the image's dimensions or pixel format. Add these additional export functions to your C++ code to expose critical metadata:

#include <opencv2/opencv.hpp>
#include <cstdint>

cv::Mat _currentFrame;

// Export raw image byte data
extern "C" __declspec(dllexport) uint8_t* GetRawImage() {
    return _currentFrame.data;
}

// Export image width (columns in OpenCV)
extern "C" __declspec(dllexport) int GetImageWidth() {
    return _currentFrame.cols;
}

// Export image height (rows in OpenCV)
extern "C" __declspec(dllexport) int GetImageHeight() {
    return _currentFrame.rows;
}

// Export number of color channels (3 = BGR, 4 = BGRA)
extern "C" __declspec(dllexport) int GetImageChannels() {
    return _currentFrame.channels();
}

This gives Unity all the context it needs to correctly interpret the raw byte stream.

Step 2: Unity C# Implementation #

Now write a C# script that imports these DLL functions, fetches the frame data, and converts it to a Texture2D for display. We'll handle color format conversion (OpenCV uses BGR by default, Unity expects RGB) and real-time updates:

using System;
using System.Runtime.InteropServices;
using UnityEngine;
using UnityEngine.UI;

public class OpenCVFrameRenderer : MonoBehaviour
{
    // Replace with your actual DLL name/path
    private const string OpenCVDllName = "YourOpenCVPlugin";

    [DllImport(OpenCVDllName)]
    private static extern IntPtr GetRawImage();

    [DllImport(OpenCVDllName)]
    private static extern int GetImageWidth();

    [DllImport(OpenCVDllName)]
    private static extern int GetImageHeight();

    [DllImport(OpenCVDllName)]
    private static extern int GetImageChannels();

    private Texture2D _frameTexture;

    void Start()
    {
        // Initialize texture with the first frame's dimensions
        int width = GetImageWidth();
        int height = GetImageHeight();
        int channels = GetImageChannels();

        if (width <= 0 || height <= 0 || (channels != 3 && channels != 4))
        {
            Debug.LogError("Invalid frame data from OpenCV! Check your camera/processing setup.");
            return;
        }

        // Match Unity's texture format to OpenCV's channel count
        TextureFormat textureFormat = channels == 3 ? TextureFormat.RGB24 : TextureFormat.BGRA32;
        _frameTexture = new Texture2D(width, height, textureFormat, false);
        _frameTexture.wrapMode = TextureWrapMode.Clamp;

        // Assign to a RawImage UI element (adjust this to match your display setup)
        RawImage display = GetComponent<RawImage>();
        if (display != null)
        {
            display.texture = _frameTexture;
            display.uvRect = new Rect(0, 1, 1, -1); // Flip vertically if needed (OpenCV frames are often upside-down)
        }
    }

    void Update()
    {
        if (_frameTexture == null) return;

        // Check if frame dimensions changed (e.g., camera resolution switch)
        int currentWidth = GetImageWidth();
        int currentHeight = GetImageHeight();
        int currentChannels = GetImageChannels();

        if (_frameTexture.width != currentWidth || _frameTexture.height != currentHeight)
        {
            // Destroy old texture and create a new one with updated dimensions
            Destroy(_frameTexture);
            TextureFormat textureFormat = currentChannels == 3 ? TextureFormat.RGB24 : TextureFormat.BGRA32;
            _frameTexture = new Texture2D(currentWidth, currentHeight, textureFormat, false);
            GetComponent<RawImage>().texture = _frameTexture;
        }

        // Fetch raw frame data from the DLL
        IntPtr frameDataPtr = GetRawImage();
        if (frameDataPtr == IntPtr.Zero)
        {
            Debug.LogWarning("No frame data available from OpenCV.");
            return;
        }

        // Copy bytes from unmanaged C++ memory to managed C# array
        int totalBytes = currentWidth * currentHeight * currentChannels;
        byte[] frameBytes = new byte[totalBytes];
        Marshal.Copy(frameDataPtr, frameBytes, 0, totalBytes);

        // Convert OpenCV's BGR format to Unity's RGB (only needed for 3-channel images)
        if (currentChannels == 3)
        {
            for (int i = 0; i < frameBytes.Length; i += 3)
            {
                // Swap blue and red channels
                byte temp = frameBytes[i];
                frameBytes[i] = frameBytes[i + 2];
                frameBytes[i + 2] = temp;
            }
        }

        // Update the texture with new frame data
        _frameTexture.LoadRawTextureData(frameBytes);
        _frameTexture.Apply();
    }
}

Key Notes for Reliability & Performance #

• Memory Safety: Ensure _currentFrame in your C++ code remains valid while Unity is accessing its data. If you're updating the frame in real-time, use a mutex or double-buffering to avoid race conditions.
• Color Format Optimization: Instead of converting channels in C#, do it directly in OpenCV with cv::cvtColor(_currentFrame, _currentFrame, cv::COLOR_BGR2RGB) or cv::COLOR_BGR2BGRA. This reduces CPU overhead in Unity.
• Vertical Flip: OpenCV frames are often stored upside-down relative to Unity's texture coordinate system. The uvRect adjustment in the Start() method fixes this—remove it if your frame displays correctly without it.
• Error Handling: Add checks for null pointers and invalid dimensions to avoid crashes if OpenCV fails to capture a frame.

内容的提问来源于stack exchange，提问作者Nicolas Caous