如何在iOS上通过MobileVLCKit捕获预渲染视频以获取实时Base64图像

核心思路是利用VLC的smem（内存流）模块，将播放的视频帧输出到内存缓冲区，通过C层回调获取帧数据，再转换为UIImage并生成Base64字符串。

Swift 实现步骤 #

1. 配置VLC媒体选项与回调 #

初始化VLCMedia时添加smem参数，指定回调函数、像素格式等：

import MobileVLCKit
import Accelerate

// C风格回调函数，接收VLC输出的帧数据
@convention(c) func smemCallback(opaque: UnsafeMutableRawPointer?, buf: UnsafeMutablePointer<UInt8>?, len: Int, pts: Int64, flags: UInt32, format: UnsafePointer<VLCSmemFormat>?) -> Int32 {
    guard let buf = buf, let format = format else { return 0 }
    
    let width = Int(format.pointee.width)
    let height = Int(format.pointee.height)
    
    // YUV420P转ARGB8888
    let yuvBuffer = vImage_Buffer(data: buf, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: width * 3)
    var rgbBuffer = vImage_Buffer()
    rgbBuffer.data = malloc(width * height * 4)
    rgbBuffer.height = vImagePixelCount(height)
    rgbBuffer.width = vImagePixelCount(width)
    rgbBuffer.rowBytes = width * 4
    
    let converter = vImageConverter_YUV420ToARGB8888()
    vImageConvert_AnyToAny(&yuvBuffer, &rgbBuffer, nil, converter, nil, 0, vImage_Flags(kvImagePrintDiagnosticsToConsole))
    
    // 生成UIImage
    let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue)
    guard let context = CGContext(data: rgbBuffer.data, width: width, height: height, bitsPerComponent: 8, bytesPerRow: rgbBuffer.rowBytes, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo.rawValue),
          let cgImage = context.makeImage() else {
        free(rgbBuffer.data)
        return 0
    }
    let image = UIImage(cgImage: cgImage)
    
    // 转换为Base64字符串
    if let imageData = image.jpegData(compressionQuality: 0.8) {
        let base64String = imageData.base64EncodedString()
        // 回到主线程处理业务逻辑
        DispatchQueue.main.async {
            print(base64String)
            // 这里可添加UI更新、网络请求等操作
        }
    }
    
    free(rgbBuffer.data)
    return 0
}

// 初始化VLC播放器
func setupVLCPlayer() {
    let player = VLCMediaPlayer()
    let mediaURL = URL(string: "你的视频地址")!
    let media = VLCMedia(url: mediaURL)
    
    // 配置smem参数
    let smemOptions = [
        ":sout=#smem:",
        ":sout-smem-video-preroll",
        ":sout-smem-video-callback=\(unsafeBitCast(smemCallback, to: Int.self))",
        ":sout-smem-video-format=RV24"
    ]
    media.addOptions(smemOptions)
    
    player.media = media
    player.play()
}

注意事项 #

• 需导入Accelerate框架处理像素格式转换
• 回调运行在VLC后台线程，UI操作必须切换到主线程
• 可根据视频实际像素格式调整转换逻辑，若用RGB格式可跳过YUV转RGB步骤

Objective-C 实现步骤 #

1. 配置VLC媒体选项与回调 #

#import <MobileVLCKit/MobileVLCKit.h>
#import <Accelerate/Accelerate.h>

// C风格回调函数
int32_t smemCallback(void *opaque, uint8_t *buf, int len, int64_t pts, uint32_t flags, const VLCSmemFormat *format) {
    if (!buf || !format) return 0;
    
    NSInteger width = format->width;
    NSInteger height = format->height;
    
    // YUV420P转ARGB8888
    vImage_Buffer yuvBuffer = {
        .data = buf,
        .height = (vImagePixelCount)height,
        .width = (vImagePixelCount)width,
        .rowBytes = width * 3
    };
    
    vImage_Buffer rgbBuffer;
    rgbBuffer.data = malloc(width * height * 4);
    rgbBuffer.height = (vImagePixelCount)height;
    rgbBuffer.width = (vImagePixelCount)width;
    rgbBuffer.rowBytes = width * 4;
    
    vImageConverterRef converter = vImageConverter_YUV420ToARGB8888();
    vImageConvert_AnyToAny(&yuvBuffer, &rgbBuffer, NULL, converter, NULL, 0, kvImagePrintDiagnosticsToConsole);
    
    // 生成UIImage
    CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
    CGContextRef context = CGBitmapContextCreate(rgbBuffer.data, width, height, 8, rgbBuffer.rowBytes, colorSpace, kCGImageAlphaPremultipliedLast);
    CGImageRef cgImage = CGBitmapContextCreateImage(context);
    UIImage *image = [UIImage imageWithCGImage:cgImage];
    
    // 转换为Base64字符串
    NSData *imageData = UIImageJPEGRepresentation(image, 0.8);
    NSString *base64String = [imageData base64EncodedStringWithOptions:0];
    
    // 主线程处理业务逻辑
    dispatch_async(dispatch_get_main_queue(), ^{
        NSLog(@"%@", base64String);
        // 这里可添加UI更新、网络请求等操作
    });
    
    // 释放资源
    CGImageRelease(cgImage);
    CGContextRelease(context);
    CGColorSpaceRelease(colorSpace);
    free(rgbBuffer.data);
    vImageConverterRelease(converter);
    
    return 0;
}

// 初始化VLC播放器
- (void)setupVLCPlayer {
    VLCMediaPlayer *player = [[VLCMediaPlayer alloc] init];
    NSURL *mediaURL = [NSURL URLWithString:@"你的视频地址"];
    VLCMedia *media = [[VLCMedia alloc] initWithURL:mediaURL];
    
    // 配置smem参数
    NSArray *smemOptions = @[
        @":sout=#smem:",
        @":sout-smem-video-preroll",
        [NSString stringWithFormat:@":sout-smem-video-callback=%p", smemCallback],
        @":sout-smem-video-format=RV24"
    ];
    [media addOptions:smemOptions];
    
    player.media = media;
    [player play];
}

注意事项 #

• 确保项目已导入MobileVLCKit和Accelerate框架
• 回调中需严谨管理内存，避免泄漏
• 可根据视频像素格式调整转换逻辑，若使用RGB格式可简化流程

内容的提问来源于stack exchange，提问作者Tan Pham