React Native Vision Camera帧处理器：自定义返回Image/ImageProxy

核心思路 #

要替换默认的ImageProxy（Android）/Frame（iOS），核心是将加载的图片转换成相机输出格式兼容的像素缓冲区，再包装成Vision Camera能识别的帧对象返回。

Android 端实现 #

步骤 #

1. 加载图片为Bitmap（你已实现此步骤）
2. 将Bitmap转换为Image对象（匹配相机YUV_420_888格式）
3. 包装成自定义ImageProxy返回，并关闭原帧避免内存泄漏

示例代码 #

import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.ImageFormat
import android.graphics.Matrix
import android.graphics.Rect
import android.media.Image
import android.media.ImageReader
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
import com.mrousavy.camera.frameprocessor.FrameProcessorRegistry
import java.nio.ByteBuffer

class CustomFrameProcessor : FrameProcessorPlugin {
    override fun callback(frame: ImageProxy, arguments: Map<String, Any>?): Any? {
        // 加载目标图片
        val loadedBitmap: Bitmap? = arguments?.get("imagePath")?.let {
            BitmapFactory.decodeFile(it as String)
        } ?: return frame

        // 创建ImageReader生成兼容格式的Image
        val imageReader = ImageReader.newInstance(
            loadedBitmap.width,
            loadedBitmap.height,
            ImageFormat.YUV_420_888,
            1
        )
        val canvas = imageReader.surface.lockCanvas(null)
        canvas.drawBitmap(loadedBitmap, Matrix(), null)
        imageReader.surface.unlockCanvasAndPost(canvas)
        val targetImage = imageReader.acquireNextImage()

        // 关闭原帧释放资源
        frame.close()

        // 构造自定义ImageProxy返回
        return object : ImageProxy {
            override fun getImage(): Image = targetImage
            override fun getWidth(): Int = loadedBitmap.width
            override fun getHeight(): Int = loadedBitmap.height
            override fun getFormat(): Int = targetImage.format
            override fun getCropRect(): Rect = targetImage.cropRect
            override fun setCropRect(cropRect: Rect?) { targetImage.cropRect = cropRect!! }
            override fun close() = targetImage.close()
            override fun getPlanes(): Array<ImageProxy.PlaneProxy> {
                return targetImage.planes.map { plane ->
                    object : ImageProxy.PlaneProxy {
                        override fun getBuffer(): ByteBuffer = plane.buffer
                        override fun getRowStride(): Int = plane.rowStride
                        override fun getPixelStride(): Int = plane.pixelStride
                    }
                }.toTypedArray()
            }
        }
    }
}

// 注册帧处理器插件
FrameProcessorRegistry.registerPlugin("customFrameProcessor", ::CustomFrameProcessor)

iOS 端实现 #

步骤 #

1. 加载图片为UIImage
2. 将UIImage转换为CVPixelBuffer（匹配相机kCVPixelFormatType_420YpCbCr8BiPlanarFullRange格式）
3. 包装成Frame对象返回，关闭原帧释放资源

示例代码 #

import VisionCamera
import UIKit
import CoreVideo

class CustomFrameProcessor: FrameProcessorPlugin {
    static func callback(_ frame: Frame, _ arguments: [String: Any]?) -> Any? {
        // 加载目标图片
        guard let imagePath = arguments?["imagePath"] as? String,
              let loadedImage = UIImage(contentsOfFile: imagePath) else {
            return frame
        }

        // 转换UIImage为兼容的CVPixelBuffer
        guard let targetPixelBuffer = convertToCameraPixelBuffer(loadedImage) else {
            return frame
        }

        // 关闭原帧释放资源
        frame.close()

        // 构造新Frame返回
        return Frame(
            pixelBuffer: targetPixelBuffer,
            orientation: frame.orientation,
            timestamp: frame.timestamp
        )
    }

    private static func convertToCameraPixelBuffer(_ image: UIImage) -> CVPixelBuffer? {
        let width = Int(image.size.width)
        let height = Int(image.size.height)
        let pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange

        let attributes: [CFString: Any] = [
            kCVPixelBufferCGImageCompatibilityKey: true,
            kCVPixelBufferCGBitmapContextCompatibilityKey: true,
            kCVPixelBufferPixelFormatTypeKey: pixelFormat
        ]

        var pixelBuffer: CVPixelBuffer?
        let status = CVPixelBufferCreate(
            kCFAllocatorDefault,
            width,
            height,
            pixelFormat,
            attributes as CFDictionary,
            &pixelBuffer
        )

        guard status == kCVReturnSuccess, let buffer = pixelBuffer else { return nil }

        CVPixelBufferLockBaseAddress(buffer, [])
        defer { CVPixelBufferUnlockBaseAddress(buffer, []) }

        guard let context = CGContext(
            data: CVPixelBufferGetBaseAddress(buffer),
            width: width,
            height: height,
            bitsPerComponent: 8,
            bytesPerRow: CVPixelBufferGetBytesPerRow(buffer),
            space: CGColorSpaceCreateDeviceRGB(),
            bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue
        ) else { return nil }

        context.draw(image.cgImage!, in: CGRect(x: 0, y: 0, width: width, height: height))
        return buffer
    }
}

// 注册帧处理器插件
FrameProcessorRegistry.shared.registerPlugin(name: "customFrameProcessor") { _ in
    CustomFrameProcessor()
}

React Native 端调用示例 #

import { Camera, useFrameProcessor } from 'react-native-vision-camera';
import { StyleSheet } from 'react-native';

function CameraScreen() {
  const frameProcessor = useFrameProcessor((frame) => {
    'worklet';
    // 调用自定义帧处理器，传入目标图片路径
    return customFrameProcessor(frame, { 
      imagePath: '/your/local/image/path.jpg' 
    });
  }, []);

  return (
    <Camera
      style={StyleSheet.absoluteFill}
      device={/* 你的相机设备对象 */}
      isActive={true}
      frameProcessor={frameProcessor}
      frameProcessorFps={30}
    />
  );
}

关键注意事项 #

• 格式匹配：必须保证转换后的帧格式与相机输出格式一致，否则会出现画面异常或崩溃。
• 内存管理：务必关闭原ImageProxy/Frame，避免内存泄漏。
• 性能优化：高帧率场景下建议缓存转换后的像素缓冲区，减少重复转换开销。

内容的提问来源于stack exchange，提问作者Nicolas Romano