在VideoSink中实现YUV NV21转RGB的问题求助

你当前的问题是仅处理了Y（亮度）分量，未正确解析UV（色度）分量，导致输出黑白图像。以下提供两种解决方案，适配你的代码场景：

方案一：手动实现NV21转RGB（不依赖OpenCV） #

修正YuvToRgbVideoSink的转换逻辑，正确解析NV21的VU色度数据，并将RGB数据渲染到SurfaceViewRenderer：

class YuvToRgbVideoSink(private val surfaceViewRenderer: SurfaceViewRenderer) : VideoSink {
    private var rgbBuffer: IntArray? = null
    private var targetBitmap: Bitmap? = null

    override fun onFrame(videoFrame: VideoFrame) {
        val buffer = videoFrame.buffer.toI420()
        val width = buffer.width
        val height = buffer.height

        // 初始化缓存数组
        if (rgbBuffer == null || rgbBuffer!!.size != width * height) {
            rgbBuffer = IntArray(width * height)
        }
        if (targetBitmap == null || targetBitmap!!.width != width || targetBitmap!!.height != height) {
            targetBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
        }

        // 将WebRTC的I420格式转为NV21
        val nv21Data = convertI420ToNV21(buffer)
        // NV21转RGB
        decodeYUV420SP(rgbBuffer!!, nv21Data, width, height)
        // 设置Bitmap像素并渲染
        targetBitmap!!.setPixels(rgbBuffer!!, 0, width, 0, 0, width, height)
        surfaceViewRenderer.post {
            surfaceViewRenderer.drawBitmap(targetBitmap!!, null, surfaceViewRenderer.bounds)
        }

        // 释放buffer资源
        buffer.release()
    }

    // I420转NV21格式
    private fun convertI420ToNV21(buffer: I420Buffer): ByteArray {
        val width = buffer.width
        val height = buffer.height
        val ySize = width * height
        val nv21 = ByteArray(ySize + ySize / 2)

        // 复制Y平面数据
        System.arraycopy(buffer.y, 0, nv21, 0, ySize)
        // 合并V、U平面为NV21的VU交替格式
        var uvIndex = ySize
        for (i in 0 until height / 2) {
            for (j in 0 until width / 2) {
                nv21[uvIndex++] = buffer.v[i * buffer.vStride + j]
                nv21[uvIndex++] = buffer.u[i * buffer.uStride + j]
            }
        }
        return nv21
    }

    // NV21转RGB核心转换逻辑
    fun decodeYUV420SP(rgb: IntArray, yuv420sp: ByteArray, width: Int, height: Int) {
        val frameSize = width * height
        var yp = 0
        for (j in 0 until height) {
            var uvp = frameSize + (j shr 1) * width
            var u = 0
            var v = 0
            for (i in 0 until width) {
                // 解析Y分量，做偏移校正
                var y = (0xff and yuv420sp[yp].toInt()) - 16
                y = y.coerceAtLeast(0)

                // 每两个Y像素共享一组UV分量
                if (i % 2 == 0) {
                    v = (0xff and yuv420sp[uvp++].toInt()) - 128
                    u = (0xff and yuv420sp[uvp++].toInt()) - 128
                }

                // YUV转RGB标准公式（BT.601色彩空间）
                val y1192 = 1192 * y
                var r = y1192 + 1634 * v
                var g = y1192 - 833 * v - 400 * u
                var b = y1192 + 2066 * u

                // 裁剪到有效色彩范围
                r = r.coerceIn(0, 262143)
                g = g.coerceIn(0, 262143)
                b = b.coerceIn(0, 262143)

                // 转换为ARGB格式
                rgb[yp] = -0x1000000 or ((r shr 6) shl 16) or ((g shr 6) shl 8) or (b shr 6)
                yp++
            }
        }
    }
}

方案二：使用OpenCV 4.6.0快速转换 #

利用OpenCV内置的色彩转换函数，避免手动编写转换逻辑，减少出错概率：

class YuvToRgbVideoSink(private val surfaceViewRenderer: SurfaceViewRenderer) : VideoSink {
    override fun onFrame(videoFrame: VideoFrame) {
        val buffer = videoFrame.buffer.toI420()
        val width = buffer.width
        val height = buffer.height

        // 将I420转为NV21格式
        val nv21Data = convertI420ToNV21(buffer)
        // OpenCV将NV21转为RGB
        val yuvMat = Mat(height * 3 / 2, width, CvType.CV_8UC1, nv21Data)
        val rgbMat = Mat()
        Imgproc.cvtColor(yuvMat, rgbMat, Imgproc.COLOR_YUV2RGB_NV21)

        // Mat转Bitmap并渲染
        val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
        Utils.matToBitmap(rgbMat, bitmap)
        surfaceViewRenderer.post {
            surfaceViewRenderer.drawBitmap(bitmap, null, surfaceViewRenderer.bounds)
        }

        // 释放所有资源
        buffer.release()
        yuvMat.release()
        rgbMat.release()
    }

    // I420转NV21格式（同方案一）
    private fun convertI420ToNV21(buffer: I420Buffer): ByteArray {
        val width = buffer.width
        val height = buffer.height
        val ySize = width * height
        val nv21 = ByteArray(ySize + ySize / 2)

        System.arraycopy(buffer.y, 0, nv21, 0, ySize)
        var uvIndex = ySize
        for (i in 0 until height / 2) {
            for (j in 0 until width / 2) {
                nv21[uvIndex++] = buffer.v[i * buffer.vStride + j]
                nv21[uvIndex++] = buffer.u[i * buffer.uStride + j]
            }
        }
        return nv21
    }
}

注意事项 #

• 若你的视频采集器直接输出NV21格式，可跳过convertI420ToNV21步骤，直接通过(videoFrame.buffer as YuvBuffer).data获取原始NV21数据
• OpenCV需提前集成：在Application中初始化OpenCVLoader.initDebug()（调试环境），或使用异步初始化方式

内容的提问来源于stack exchange，提问作者Nikola C