WebRTC iOS：如何将RTCVideoFrame转为CVPixelBuffer并修改摄像头流？

我之前也碰到过一模一样的情况，WebRTC移除了直接获取CVPixelBuffer的API后，确实得换个思路处理。下面分享两种可行的方案，一种是实现RTCVideoFrame到CVPixelBuffer的转换，另一种是更高效的视频流修改方式：

一、将RTCVideoFrame转为CVPixelBuffer的可行方法 #

WebRTC现在的帧缓冲基本都是基于YUV格式（主要是I420），我们可以先把RTCVideoFrameBuffer转换成RTCI420Buffer，再手动将YUV数据拷贝到新创建的CVPixelBuffer中。具体代码如下：

func convertRTCVideoFrameToCVPixelBuffer(_ frame: RTCVideoFrame) -> CVPixelBuffer? {
    // 先将帧缓冲转为I420格式
    guard let i420Buffer = frame.buffer.toI420() as? RTCI420Buffer else {
        print("无法转换为I420缓冲")
        return nil
    }
    
    let width = i420Buffer.width
    let height = i420Buffer.height
    
    // 创建CVPixelBuffer，指定YUV420BiPlanar格式
    var pixelBuffer: CVPixelBuffer?
    let creationStatus = CVPixelBufferCreate(
        kCFAllocatorDefault,
        width,
        height,
        kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
        [
            kCVPixelBufferMetalCompatibilityKey: true,
            kCVPixelBufferCGImageCompatibilityKey: true
        ] as CFDictionary,
        &pixelBuffer
    )
    
    guard creationStatus == kCVReturnSuccess, let pb = pixelBuffer else {
        print("创建CVPixelBuffer失败")
        return nil
    }
    
    CVPixelBufferLockBaseAddress(pb, .readWrite)
    
    // 拷贝Y平面数据
    if let yDestBase = CVPixelBufferGetBaseAddressOfPlane(pb, 0) {
        memcpy(yDestBase, i420Buffer.dataY, i420Buffer.strideY * height)
    }
    
    // 拷贝UV平面数据（I420的U/V是单独平面，需要转为交错格式）
    if let uvDestBase = CVPixelBufferGetBaseAddressOfPlane(pb, 1) {
        let uvStride = CVPixelBufferGetBytesPerRowOfPlane(pb, 1)
        var uvByteIndex = 0
        
        for row in 0..<height/2 {
            let uRowData = i420Buffer.dataU + row * i420Buffer.strideU
            let vRowData = i420Buffer.dataV + row * i420Buffer.strideV
            
            // 交错写入U和V数据
            for col in 0..<width/2 {
                uvDestBase.advanced(by: uvByteIndex).pointee = uRowData.advanced(by: col).pointee
                uvByteIndex += 1
                uvDestBase.advanced(by: uvByteIndex).pointee = vRowData.advanced(by: col).pointee
                uvByteIndex += 1
            }
            
            // 跳过每行的填充字节（如果stride和实际宽度不一致）
            uvByteIndex += uvStride - width
        }
    }
    
    CVPixelBufferUnlockBaseAddress(pb, .readWrite)
    return pb
}

二、修改RTCCameraVideoCapturer视频流的更优方案 #

直接修改SDK肯定不是最优解，我们可以通过包装原生的RTCVideoCapturerDelegate来实现视频流的拦截和修改，处理完后再转发给WebRTC的内部逻辑。这里分两种场景：

1. 基于YUV数据直接处理（性能最优） #

如果你的滤镜可以直接操作YUV格式的数据，建议直接修改RTCI420Buffer，避免格式转换的开销：

class FilteredVideoCapturerDelegate: NSObject, RTCVideoCapturerDelegate {
    private let originalDelegate: RTCVideoCapturerDelegate
    
    init(originalDelegate: RTCVideoCapturerDelegate) {
        self.originalDelegate = originalDelegate
        super.init()
    }
    
    func capturer(_ capturer: RTCVideoCapturer, didCapture frame: RTCVideoFrame) {
        guard let processedFrame = applyYUVFilter(to: frame) else {
            originalDelegate.capturer(capturer, didCapture: frame)
            return
        }
        originalDelegate.capturer(capturer, didCapture: processedFrame)
    }
    
    private func applyYUVFilter(to frame: RTCVideoFrame) -> RTCVideoFrame? {
        guard let i420Buffer = frame.buffer.toI420() as? RTCI420Buffer else {
            return nil
        }
        
        // 创建可写的I420缓冲（原缓冲通常是只读的）
        let mutableBuffer = RTCI420MutableBuffer(width: i420Buffer.width, height: i420Buffer.height)
        mutableBuffer.dataY.copyMemory(from: i420Buffer.dataY, count: i420Buffer.strideY * i420Buffer.height)
        mutableBuffer.dataU.copyMemory(from: i420Buffer.dataU, count: i420Buffer.strideU * i420Buffer.height / 2)
        mutableBuffer.dataV.copyMemory(from: i420Buffer.dataV, count: i420Buffer.strideV * i420Buffer.height / 2)
        
        // 示例：调整Y通道亮度
        let yData = mutableBuffer.dataY
        for i in 0..<(mutableBuffer.strideY * mutableBuffer.height) {
            yData[i] = min(255, max(0, yData[i] + 30))
        }
        
        // 创建处理后的视频帧
        let processedBuffer = mutableBuffer as RTCVideoFrameBuffer
        return RTCVideoFrame(buffer: processedBuffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs)
    }
}

使用时，替换原capturer的delegate即可：

// 假设原capturer是rtccameraCapturer，原delegate是self
let filteredDelegate = FilteredVideoCapturerDelegate(originalDelegate: self)
rtccameraCapturer.delegate = filteredDelegate

2. 基于CoreImage等CVPixelBuffer的滤镜处理 #

如果需要用CoreImage这类依赖CVPixelBuffer的框架，就可以先把RTCVideoFrame转成CVPixelBuffer，处理后再转回RTCVideoFrame：

private func applyCoreImageFilter(to frame: RTCVideoFrame) -> RTCVideoFrame? {
    guard let inputPixelBuffer = convertRTCVideoFrameToCVPixelBuffer(frame) else {
        return nil
    }
    
    // 应用CoreImage滤镜（示例：棕褐色滤镜）
    let ciImage = CIImage(cvImageBuffer: inputPixelBuffer)
    guard let filter = CIFilter(name: "CISepiaTone") else {
        return nil
    }
    filter.setValue(ciImage, forKey: kCIInputImageKey)
    filter.setValue(0.8, forKey: kCIInputIntensityKey)
    
    guard let filteredImage = filter.outputImage else {
        return nil
    }
    
    // 将处理后的CIImage转回CVPixelBuffer
    let context = CIContext(options: [.useSoftwareRenderer: false])
    var outputPixelBuffer: CVPixelBuffer?
    let bufferAttributes = [
        kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
        kCVPixelBufferWidthKey: filteredImage.extent.width,
        kCVPixelBufferHeightKey: filteredImage.extent.height
    ] as CFDictionary
    
    CVPixelBufferCreate(kCFAllocatorDefault,
                        Int(filteredImage.extent.width),
                        Int(filteredImage.extent.height),
                        kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
                        bufferAttributes,
                        &outputPixelBuffer)
    
    guard let pb = outputPixelBuffer else {
        return nil
    }
    
    context.render(filteredImage, to: pb)
    
    // 将CVPixelBuffer转回RTCVideoFrame
    let rtcBuffer = RTCCVPixelBuffer(pixelBuffer: pb)
    return RTCVideoFrame(buffer: rtcBuffer, rotation: frame.rotation, timeStampNs: frame.timeStampNs)
}

然后把这个方法替换到上面的FilteredVideoCapturerDelegate的applyYUVFilter位置即可。

总结 #

• 如果只是需要转CVPixelBuffer，通过I420缓冲手动拷贝YUV数据是目前可行的方案；
• 修改视频流优先选择直接操作YUV缓冲的方式，性能更好；如果必须用CoreImage等框架，再进行格式转换；
• 通过包装delegate的方式，不需要修改WebRTC SDK，完全可以在Xcode工程内实现。

内容的提问来源于stack exchange，提问作者Giraff Wombat