如何从已连接的GoPro相机获取视频帧缓冲流并在iOS中使用？

我正在开发一款可连接GoPro相机的iOS应用，目标是从已连接的GoPro获取视频帧缓冲流，转换为CMSampleBuffer后通过AVSampleBufferDisplayLayer显示。目前尝试使用FFmpegKit处理UDP流，但未成功显示画面，以下是实现代码和FFmpeg日志信息：

现有实现代码 #

import Foundation
import CoreMedia
import AVFoundation
import ffmpegkit


final class FFmpegBufferProcessor: AnyBufferProcessor {
    weak var delegate: BufferProcessorDelegate?
    
    private var pipePath: String = NSTemporaryDirectory() + "ffmpeg_pipe"
    private var isProcessing: Bool = false
    private var videoWidth = 1920
    private var videoHeight = 1080
    private let pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
    
    init() {
        setupPipe()
    }
    
    deinit {
        cleanupPipe()
    }
    
    private func setupPipe() {
        do {
            if FileManager.default.fileExists(atPath: pipePath) {
                try FileManager.default.removeItem(atPath: pipePath)
            }
            
            let result = mkfifo(pipePath.cString(using: .utf8), 0o644)
            if result != 0 {
                print("\(#function); Pipe creation failed.")
                return
            }
        } catch {
            print("\(#function); Setup pipe error: \(error.localizedDescription)")
        }
    }
    
    private func cleanupPipe() {
        do {
            try FileManager.default.removeItem(atPath: pipePath)
        } catch {
            print("\(#function); Cleanup pipe error: \(error.localizedDescription)")
        }
    }
    
    func startProcessingStream(from udpURL: String) {
        guard !isProcessing else {
            print("\(#function); Already processing stream.")
            return
        }
        
        isProcessing = true
        let command = """        -i \(udpURL) -f rawvideo -pix_fmt nv12 \(pipePath)
        """
        
        FFmpegKit.executeAsync(command) { [weak self] session in
            let returnCode = session?.getReturnCode()
            if ReturnCode.isSuccess(returnCode) {
                print("\(#function); FFmpeg session completed.")
            } else {
                print("\(#function); FFmpeg session error: \(String(describing: session?.getFailStackTrace())).")
            }
            
            self?.isProcessing = false
        }
        
        readFromPipe()
    }
    
    func stopProcessingStream() {
        isProcessing = false
        FFmpegKit.cancel()
    }
}

// MARK: - Private methods

private extension FFmpegBufferProcessor {
    func readFromPipe() {
        DispatchQueue.global(qos: .background).async { [unowned self] in
            guard let fileHandle = FileHandle(forReadingAtPath: self.pipePath) else {
                print("\(#function); Fail to read file handle from pipe path.")
                return
            }
            
            autoreleasepool {
                while self.isProcessing {
                    let frameSize = self.videoWidth * self.videoHeight * 3 / 2
                    let rawData = fileHandle.readData(ofLength: frameSize)
                    
                    if rawData.isEmpty {
                        print("\(#function); Pipe closed / no more data to read.")
                        break
                    }
                    
                    self.handleRawFrameData(rawData)
                }
                
                fileHandle.closeFile()
            }
        }
    }
    
    func handleRawFrameData(_ data: Data) {
        let width = 1920
        let height = 1080
        
        // Creating the Pixel Buffer (if possible)
        guard let pixelBuffer = createPixelBuffer(from: data, width: width, height: height) else {
            print("\(#function); Failed to create pixel buffer")
            return
        }
        
        var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: 30), presentationTimeStamp: .zero, decodeTimeStamp: .invalid)
        // Creating the Sample Buffer (if possible)
        guard let sampleBuffer = createSampleBuffer(from: pixelBuffer, timing: &timing) else {
            print("\(#function); Failed to create sample buffer")
            return
        }
        
        delegate?.bufferProcessor(self, didOutput: sampleBuffer)
    }
}

FFmpeg日志信息 #

日志显示关键错误点：

• 成功识别UDP流为H.264编码、1920x1080分辨率、30fps帧率
• 出现Invalid UE golomb code解码错误，后续触发Error while decoding stream #0:0: Invalid data found when processing input
• 最终FFmpeg进程异常终止，提示Conversion failed!

当前问题 #

AVSampleBufferDisplayLayer入队CMSampleBuffer后无画面显示，且FFmpeg处理流时持续出现解码失败。

一、修复现有FFmpeg实现问题 #

1. 修正FFmpeg命令参数 #

当前命令未针对GoPro UDP流的封装格式做适配，添加硬件加速和缓冲区参数解决解码错误：

let command = """
-i \(udpURL) \
-c:v h264_videotoolbox \
-f rawvideo \
-pix_fmt nv12 \
-r 30 \
-buffer_size 1024000 \
\(pipePath)
"""

• -c:v h264_videotoolbox：使用iOS硬件加速解码，提升效率并减少软解码的兼容性问题
• -buffer_size 1024000：增大输入缓冲区，处理UDP流的丢包、乱序问题
• -r 30：强制指定输出帧率，避免FFmpeg自动识别帧率出错

2. 修复CMSampleBuffer时间戳问题 #

固定时间戳会导致AVSampleBufferDisplayLayer无法正确渲染帧序列，需生成递增的时间戳：

// 新增成员变量记录帧计数
private var frameCount: Int64 = 0

func handleRawFrameData(_ data: Data) {
    let width = 1920
    let height = 1080
    
    guard let pixelBuffer = createPixelBuffer(from: data, width: width, height: height) else {
        print("\(#function); Failed to create pixel buffer")
        return
    }
    
    let frameDuration = CMTime(value: 1, timescale: 30)
    let presentationTime = CMTimeMultiply(frameDuration, multiplier: frameCount)
    var timing = CMSampleTimingInfo(
        duration: frameDuration,
        presentationTimeStamp: presentationTime,
        decodeTimeStamp: .invalid
    )
    
    guard let sampleBuffer = createSampleBuffer(from: pixelBuffer, timing: &timing) else {
        print("\(#function); Failed to create sample buffer")
        return
    }
    
    frameCount += 1
    delegate?.bufferProcessor(self, didOutput: sampleBuffer)
}

3. 补充缺失的createPixelBuffer和createSampleBuffer实现 #

当前代码缺少核心的像素缓冲区和样本缓冲区创建逻辑，补充如下：

private func createPixelBuffer(from data: Data, width: Int, height: Int) -> CVPixelBuffer? {
    let attributes: [CFString: Any] = [
        kCVPixelBufferWidthKey: width,
        kCVPixelBufferHeightKey: height,
        kCVPixelBufferPixelFormatTypeKey: pixelFormat,
        kCVPixelBufferBytesPerRowAlignmentKey: width,
        kCVPixelBufferMemoryAllocatorKey: kCFAllocatorDefault
    ]
    
    var pixelBuffer: CVPixelBuffer?
    let status = CVPixelBufferCreateWithBytes(
        kCFAllocatorDefault,
        width,
        height,
        pixelFormat,
        UnsafeMutableRawPointer(mutating: (data as NSData).bytes),
        width,
        nil,
        nil,
        nil,
        &pixelBuffer
    )
    
    guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
        print("CVPixelBufferCreateWithBytes failed: \(status)")
        return nil
    }
    
    // 配置NV12格式的CbCr平面地址
    CVPixelBufferLockBaseAddress(buffer, [])
    let yPlaneSize = width * height
    let cbCrBaseAddress = CVPixelBufferGetBaseAddress(buffer)?.advanced(by: yPlaneSize)
    CVPixelBufferSetBaseAddressOfPlane(buffer, 1, cbCrBaseAddress)
    CVPixelBufferSetBytesPerRowOfPlane(buffer, 1, width)
    CVPixelBufferUnlockBaseAddress(buffer, [])
    
    return buffer
}

private func createSampleBuffer(from pixelBuffer: CVPixelBuffer, timing: inout CMSampleTimingInfo) -> CMSampleBuffer? {
    var formatDescription: CMVideoFormatDescription?
    CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescription)
    
    guard let formatDesc = formatDescription else {
        print("Failed to create format description")
        return nil
    }
    
    var sampleBuffer: CMSampleBuffer?
    let status = CMSampleBufferCreateForImageBuffer(
        allocator: kCFAllocatorDefault,
        imageBuffer: pixelBuffer,
        dataReady: true,
        makeDataReadyCallback: nil,
        refcon: nil,
        formatDescription: formatDesc,
        sampleTiming: &timing,
        sampleBufferOut: &sampleBuffer
    )
    
    guard status == kCVReturnSuccess else {
        print("CMSampleBufferCreateForImageBuffer failed: \(status)")
        return nil
    }
    
    return sampleBuffer
}

4. 保证AVSampleBufferDisplayLayer的线程安全 #

CMSampleBuffer的入队操作必须在主线程执行：

// 在delegate回调中切换到主线程
func bufferProcessor(_ processor: AnyBufferProcessor, didOutput sampleBuffer: CMSampleBuffer) {
    DispatchQueue.main.async {
        if self.displayLayer.isReadyForMoreMediaData {
            self.displayLayer.enqueue(sampleBuffer)
        }
    }
}

二、替代方案：使用成熟库或官方SDK #

1. GoPro官方SDK（GoPro Connect） #

GoPro提供官方iOS SDK，可直接获取视频流并转换为CMSampleBuffer，无需手动处理解码，稳定性和兼容性更优，可前往GoPro开发者门户获取。

2. 使用MobileVLCKit拉流解码 #

通过MobileVLCKit直接处理GoPro的UDP流，简化帧获取逻辑：

import MobileVLCKit

class VLCMediaPlayerManager: NSObject, VLCMediaPlayerDelegate {
    private let mediaPlayer = VLCMediaPlayer()
    weak var delegate: FrameOutputDelegate?
    
    func setupPlayer(with udpURL: String) {
        let media = VLCMedia(url: URL(string: udpURL)!)
        mediaPlayer.media = media
        mediaPlayer.delegate = self
        mediaPlayer.play()
    }
    
    func mediaPlayer(_ mediaPlayer: VLCMediaPlayer, didOutputPicture picture: VLCPicture) {
        // 将VLCPicture转换为CVPixelBuffer
        guard let pixelBuffer = picture.toCVPixelBuffer() else { return }
        // 生成递增时间戳并创建CMSampleBuffer
        let frameDuration = CMTime(value: 1, timescale: 30)
        let presentationTime = CMTimeMultiply(frameDuration, multiplier: frameCount)
        frameCount += 1
        // ... 封装为CMSampleBuffer并回调
        delegate?.didOutputSampleBuffer(sampleBuffer)
    }
}

内容的提问来源于stack exchange，提问作者cleanrun