iOS平台WebRTC视频通话画中画(PIP)功能实现问题求助

核心思路：通过自定义Renderer获取RTCVideoFrame #

WebRTC的RTCVideoTrack支持绑定自定义RTCVideoRenderer实现，通过实现协议方法renderFrame:可以实时获取每帧RTCVideoFrame，这是解决Track转Frame的关键。

1. 自定义视频Renderer #

#import <WebRTC/WebRTC.h>

@interface CustomVideoRenderer : NSObject <RTCVideoRenderer>
@property (nonatomic, copy) void (^frameReceivedBlock)(RTCVideoFrame *frame);
@end

@implementation CustomVideoRenderer

- (void)renderFrame:(RTCVideoFrame *)frame {
    if (self.frameReceivedBlock) {
        self.frameReceivedBlock(frame);
    }
}

@end

2. 绑定Renderer到VideoTrack #

// 获取目标RTCVideoTrack实例
RTCVideoTrack *videoTrack = ...;

// 初始化自定义Renderer并绑定帧回调
CustomVideoRenderer *renderer = [[CustomVideoRenderer alloc] init];
__weak typeof(self) weakSelf = self;
renderer.frameReceivedBlock = ^(RTCVideoFrame *frame) {
    // 切换到主线程处理UI相关操作
    dispatch_async(dispatch_get_main_queue(), ^{
        [weakSelf processVideoFrame:frame];
    });
};

// 将Renderer添加到VideoTrack，开始接收帧数据
[videoTrack addRenderer:renderer];

帧数据转换与渲染 #

3. 从RTCVideoFrame提取CVPixelBuffer #

- (void)processVideoFrame:(RTCVideoFrame *)frame {
    RTCVideoFrameBuffer *frameBuffer = frame.buffer;
    // WebRTC默认使用RTCCVPixelBuffer格式存储帧数据
    if ([frameBuffer isKindOfClass:[RTCCVPixelBuffer class]]) {
        RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frameBuffer;
        CVPixelBufferRef cvPixelBuffer = rtcPixelBuffer.pixelBuffer;
        // 手动管理引用计数，避免提前释放
        CVPixelBufferRetain(cvPixelBuffer);
        
        // 转换为CMSampleBuffer
        CMSampleBufferRef sampleBuffer = [self createCMSampleBufferFromPixelBuffer:cvPixelBuffer];
        if (sampleBuffer) {
            [self updateDisplayWithSampleBuffer:sampleBuffer];
            CMSampleBufferRelease(sampleBuffer);
        }
        CVPixelBufferRelease(cvPixelBuffer);
    }
}

4. 转换CVPixelBuffer为CMSampleBuffer #

- (CMSampleBufferRef)createCMSampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBuffer {
    // 创建视频格式描述
    CMVideoFormatDescriptionRef formatDesc = NULL;
    OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(NULL, pixelBuffer, &formatDesc);
    if (status != noErr) {
        NSLog(@"创建视频格式描述失败: %d", (int)status);
        return NULL;
    }
    
    // 设置帧时序信息（示例为30fps）
    CMSampleTimingInfo timingInfo = {
        .duration = CMTimeMake(1, 30),
        .presentationTimeStamp = CMClockGetTime(CMClockGetHostTimeClock()),
        .decodeTimeStamp = kCMTimeInvalid
    };
    
    // 创建CMSampleBuffer
    CMSampleBufferRef sampleBuffer = NULL;
    status = CMSampleBufferCreateForImageBuffer(NULL, pixelBuffer, true, NULL, NULL, formatDesc, &timingInfo, &sampleBuffer);
    CFRelease(formatDesc);
    
    if (status != noErr) {
        NSLog(@"创建CMSampleBuffer失败: %d", (int)status);
        return NULL;
    }
    
    return sampleBuffer;
}

5. 用AVSampleBufferDisplayLayer渲染帧 #

使用AVSampleBufferDisplayLayer渲染，该层支持系统PIP模式：

@interface YourViewController ()
@property (nonatomic, strong) AVSampleBufferDisplayLayer *displayLayer;
@end

@implementation YourViewController

- (void)setupDisplayLayer {
    // 初始化显示层
    self.displayLayer = [[AVSampleBufferDisplayLayer alloc] init];
    self.displayLayer.frame = self.videoContainerView.bounds;
    self.displayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
    [self.videoContainerView.layer addSublayer:self.displayLayer];
}

- (void)updateDisplayWithSampleBuffer:(CMSampleBufferRef)sampleBuffer {
    if (CMSampleBufferDataIsReady(sampleBuffer)) {
        [self.displayLayer enqueueSampleBuffer:sampleBuffer];
    }
}

@end

实现PIP功能 #

6. 集成AVPictureInPictureController #

#import <AVKit/AVKit.h>

@interface YourViewController () <AVPictureInPictureControllerDelegate>
@property (nonatomic, strong) AVPictureInPictureController *pipController;
@property (nonatomic, strong) AVPlayer *dummyPlayer;
@property (nonatomic, strong) AVPlayerLayer *playerLayer;
@end

@implementation YourViewController

- (void)setupPIPController {
    // 创建空的AVPlayer（PIP控制器依赖AVPlayerLayer）
    self.dummyPlayer = [AVPlayer playerWithURL:[NSURL URLWithString:@""]];
    self.playerLayer = [AVPlayerLayer playerLayerWithPlayer:self.dummyPlayer];
    self.playerLayer.frame = self.view.bounds;
    [self.view.layer addSublayer:self.playerLayer];
    
    // 将显示层添加到PlayerLayer
    [self.playerLayer addSublayer:self.displayLayer];
    
    // 初始化PIP控制器
    if (AVPictureInPictureController.isPictureInPictureSupported) {
        self.pipController = [[AVPictureInPictureController alloc] initWithPlayerLayer:self.playerLayer];
        self.pipController.delegate = self;
    }
}

// 启动PIP
- (void)startPictureInPicture {
    if (self.pipController && !self.pipController.isPictureInPictureActive) {
        [self.pipController startPictureInPicture];
    }
}

// PIP状态回调
- (void)pictureInPictureControllerDidStartPictureInPicture:(AVPictureInPictureController *)pictureInPictureController {
    // 隐藏本地视图或执行其他UI调整
    self.videoContainerView.hidden = YES;
}

- (void)pictureInPictureControllerDidStopPictureInPicture:(AVPictureInPictureController *)pictureInPictureController {
    // 恢复视图显示
    self.videoContainerView.hidden = NO;
}

@end

关键注意事项 #

• 内存管理：CVPixelBuffer和CMSampleBuffer需手动管理引用计数，避免内存泄漏。
• 线程安全：WebRTC帧回调在非主线程，必须切换到主线程处理UI更新。
• 性能优化：帧转换逻辑尽量放在子线程执行，避免阻塞主线程；可使用串行队列控制帧处理频率。
• 权限配置：Info.plist中需添加相机/麦克风权限描述，iPad端需将UIRequiresFullScreen设为NO以支持多任务。

内容的提问来源于stack exchange，提问作者Kavya Basavesh