如何从CameraX ImageAnalysis获取RGB帧适配Luxand FaceSDK活体检测

我正在开发基于.NET MAUI Android的应用，使用CameraX和Luxand FaceSDK。当前人脸检测流水线接收灰度图像缓冲区，能正常完成人脸检测与识别：

public List<FaceRect> ProcessFrame(byte[] gray, int width, int height, int rotationDegrees)
{
    int image;

    int rc = FSDK.LoadImageFromBuffer(
        out image,
        gray,
        width,
        height,
        width,
        FSDK.FSDK_IMAGEMODE.FSDK_IMAGE_GRAYSCALE_8BIT
    );

    ...
}

但实现Luxand活体检测时，技术支持明确说明活体检测必须使用彩色图像，灰度帧会被拒绝。当前流水线仅提取CameraX的Y平面生成灰度缓冲区，无法满足需求。

问题 #

1. 如何从CameraX ImageAnalysis获取完整的RGB图像？
2. CameraX输出YUV_420_888帧，将其转换为RGB的推荐方式是什么？
3. 转换后，应向Luxand FaceSDK传递何种缓冲区格式？
4. 是否有适用于实时人脸识别和活体检测的高性能解决方案？

环境 #

• .NET MAUI
• Android
• CameraX ImageAnalysis
• Luxand FaceSDK
• 实时人脸识别场景

当前CameraX分析器代码 #

using AndroidX.Camera.Core;
using JTClockV2.Platforms.Android.CameraX;
using System;
using Java.Nio;
using Android.App;

using Android.Util;
using ASize = Android.Util.Size;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Diagnostics;

namespace JTClockV2.Platforms.Android.CameraX
{
    public class FrameAnalyzer : Java.Lang.Object, ImageAnalysis.IAnalyzer
    {
        private readonly ICameraFrameListener _listener;

        public FrameAnalyzer(ICameraFrameListener listener)
        {
            _listener = listener;
        }
         ASize ImageAnalysis.IAnalyzer.DefaultTargetResolution
            => new ASize(640, 480);

        public void Analyze(IImageProxy image)
        {
            try
            {
                int width = image.Width;
                int height = image.Height;

                int rotationDegrees = image.ImageInfo.RotationDegrees;

                var yPlane = image.GetPlanes()[0];
                var yBuffer = yPlane.Buffer;
                int rowStride = yPlane.RowStride;

                byte[] gray = new byte[width * height];

                int pos = 0;

                for (int row = 0; row < height; row++)
                {
                    yBuffer.Position(row * rowStride);
                    yBuffer.Get(gray, pos, width);
                    pos += width;
                }

                _listener.OnFrame(gray, width, height, rotationDegrees);
            }
            finally
            {
                image.Close();
            }
        }


        private void Log(string msg, string v)
        {
            Debug.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] 🔵 LUXAND: " + msg);

        }

        private static byte[] ConvertArgbToBgra(byte[] argb)
        {
            byte[] bgra = new byte[argb.Length];

            for (int i = 0; i < argb.Length; i += 4)
            {
                byte a = argb[i];
                byte r = argb[i + 1];
                byte g = argb[i + 2];
                byte b = argb[i + 3];

                bgra[i] = b;
                bgra[i + 1] = g;
                bgra[i + 2] = r;
                bgra[i + 3] = a;
            }

            return bgra;
        }

        private static byte[] YuvToRgb24(IImageProxy image)
        {
            int width = image.Width;
            int height = image.Height;

            var yPlane = image.GetPlanes()[0];
            var uPlane = image.GetPlanes()[1];
            var vPlane = image.GetPlanes()[2];

            ByteBuffer yBuf = yPlane.Buffer;
            ByteBuffer uBuf = uPlane.Buffer;
            ByteBuffer vBuf = vPlane.Buffer;

            int yRowStride = yPlane.RowStride;
            int uvRowStride = uPlane.RowStride;
            int uvPixelStride = uPlane.PixelStride;

            byte[] rgb = new byte[width * height * 3];
            int idx = 0;

            for (int y = 0; y < height; y++)
            {
                int yOffset = y * yRowStride;
                int uvOffset = (y / 2) * uvRowStride;

                for (int x = 0; x < width; x++)
                {
                    int Y = yBuf.Get(yOffset + x) & 0xFF;
                    int U = (uBuf.Get(uvOffset + (x / 2) * uvPixelStride) & 0xFF) - 128;
                    int V = (vBuf.Get(uvOffset + (x / 2) * uvPixelStride) & 0xFF) - 128;

                    int r = (int)(Y + 1.402 * V);
                    int g = (int)(Y - 0.344 * U - 0.714 * V);
                    int b = (int)(Y + 1.772 * U);

                    rgb[idx++] = (byte)Math.Clamp(r, 0, 255);
                    rgb[idx++] = (byte)Math.Clamp(g, 0, 255);
                    rgb[idx++] = (byte)Math.Clamp(b, 0, 255);
                }
            }
            return rgb;
        }



        // 🔥 CORE CONVERSION (YUV_420_888 → RGBA)
        private static byte[] Yuv4ToRgba(IImageProxy image)
        {
            int width = image.Width;
            int height = image.Height;

            var yPlane = image.GetPlanes()[0];
            var uPlane = image.GetPlanes()[1];
            var vPlane = image.GetPlanes()[2];

            ByteBuffer yBuffer = yPlane.Buffer;
            ByteBuffer uBuffer = uPlane.Buffer;
            ByteBuffer vBuffer = vPlane.Buffer;

            int yRowStride = yPlane.RowStride;
            int uvRowStride = uPlane.RowStride;
            int uvPixelStride = uPlane.PixelStride;

            byte[] rgba = new byte[width * height * 4];

            int index = 0;

            for (int row = 0; row < height; row++)
            {
                int yRowOffset = row * yRowStride;
                int uvRowOffset = (row / 2) * uvRowStride;

                for (int col = 0; col < width; col++)
                {
                    int yIndex = yRowOffset + col;
                    int uvIndex = uvRowOffset + (col >> 1) * uvPixelStride;

                    int y = yBuffer.Get(yIndex) & 0xFF;
                    int u = (uBuffer.Get(uvIndex) & 0xFF) - 128;
                    int v = (vBuffer.Get(uvIndex) & 0xFF) - 128;

                    // YUV → RGB conversion
                    int c = y - 16;
                    int d = u - 128;
                    int e = v - 128;

                    int r = (298 * c + 409 * e + 128) >> 8;
                    int g = (298 * c - 100 * d - 208 * e + 128) >> 8;
                    int b = (298 * c + 516 * d + 128) >> 8;

                    rgba[index++] = (byte)Clamp(r);
                    rgba[index++] = (byte)Clamp(g);
                    rgba[index++] = (byte)Clamp(b);
                    rgba[index++] = 255; // Alpha
                }
            }

            return rgba;
        }

     
        private static int Clamp(int value)
        {
            if (value < 0) return 0;
            if (value > 255) return 255;
            return value;
        }
    }
}

解决方案 #

1. 从CameraX ImageAnalysis获取完整RGB图像 #

CameraX默认输出YUV_420_888格式帧，无法直接获取RGB。需要提取Y、U、V三个平面的数据，手动转换为RGB格式。修改Analyze方法，调用转换函数生成RGB缓冲区，再传递给监听者：

public void Analyze(IImageProxy image)
{
    try
    {
        int width = image.Width;
        int height = image.Height;
        int rotationDegrees = image.ImageInfo.RotationDegrees;

        // 转换YUV_420_888为RGB24格式
        byte[] rgbBuffer = YuvToRgb24(image);
        
        // 传递彩色缓冲区给处理流水线
        _listener.OnFrame(rgbBuffer, width, height, rotationDegrees);
    }
    finally
    {
        image.Close();
    }
}

2. YUV_420_888转RGB的推荐方式 #

使用你已实现的YuvToRgb24方法即可，该方法采用标准的YUV转RGB公式，计算准确且性能适中。如果追求更高性能，可以预分配缓冲区减少GC开销：

private byte[] _preallocatedRgbBuffer;

private byte[] YuvToRgb24(IImageProxy image)
{
    int width = image.Width;
    int height = image.Height;
    int bufferSize = width * height * 3;

    // 预分配缓冲区，避免重复创建
    if (_preallocatedRgbBuffer == null || _preallocatedRgbBuffer.Length != bufferSize)
    {
        _preallocatedRgbBuffer = new byte[bufferSize];
    }

    var yPlane = image.GetPlanes()[0];
    var uPlane = image.GetPlanes()[1];
    var vPlane = image.GetPlanes()[2];

    ByteBuffer yBuf = yPlane.Buffer;
    ByteBuffer uBuf = uPlane.Buffer;
    ByteBuffer vBuf = vPlane.Buffer;

    int yRowStride = yPlane.RowStride;
    int uvRowStride = uPlane.RowStride;
    int uvPixelStride = uPlane.PixelStride;

    int idx = 0;

    for (int y = 0; y < height; y++)
    {
        int yOffset = y * yRowStride;
        int uvOffset = (y / 2) * uvRowStride;

        for (int x = 0; x < width; x++)
        {
            int Y = yBuf.Get(yOffset + x) & 0xFF;
            int U = (uBuf.Get(uvOffset + (x / 2) * uvPixelStride) & 0xFF) - 128;
            int V = (vBuf.Get(uvOffset + (x / 2) * uvPixelStride) & 0xFF) - 128;

            int r = (int)(Y + 1.402 * V);
            int g = (int)(Y - 0.344 * U - 0.714 * V);
            int b = (int)(Y + 1.772 * U);

            _preallocatedRgbBuffer[idx++] = (byte)Math.Clamp(r, 0, 255);
            _preallocatedRgbBuffer[idx++] = (byte)Math.Clamp(g, 0, 255);
            _preallocatedRgbBuffer[idx++] = (byte)Math.Clamp(b, 0, 255);
        }
    }
    return _preallocatedRgbBuffer;
}

3. 传递给Luxand FaceSDK的缓冲区格式 #

推荐使用RGB24格式（每个像素3字节，顺序为R-G-B），对应SDK的FSDK_IMAGEMODE.FSDK_IMAGE_COLOR_24BIT。修改ProcessFrame方法：

public List<FaceRect> ProcessFrame(byte[] rgbBuffer, int width, int height, int rotationDegrees)
{
    int image;

    // 加载RGB24格式图像
    int rc = FSDK.LoadImageFromBuffer(
        out image,
        rgbBuffer,
        width,
        height,
        width * 3, // 每行字节数：宽度×3（RGB24每个像素3字节）
        FSDK.FSDK_IMAGEMODE.FSDK_IMAGE_COLOR_24BIT
    );

    if (rc != FSDK.FSDKE_OK)
    {
        // 处理加载失败逻辑
        return new List<FaceRect>();
    }

    // 执行人脸检测
    List<FaceRect> faces = new List<FaceRect>();
    // ... 原有检测逻辑 ...

    // 执行活体检测
    bool isLive = false;
    rc = FSDK.IsFaceLive(image, out isLive);
    if (rc == FSDK.FSDKE_OK && isLive)
    {
        // 活体检测通过逻辑
    }

    FSDK.FreeImage(image);
    return faces;
}

4. 实时场景的高性能解决方案 #

1. 降低分辨率：使用640×480或更低分辨率，减少转换和计算量
2. 限制帧率：将CameraX ImageAnalysis的帧率设置为15-20fps，避免不必要的帧处理
3. 硬件加速转换：使用Android RenderScript进行YUV转RGB，性能比纯CPU转换提升2-3倍
4. 异步处理：将图像转换和SDK处理放在后台线程，避免阻塞UI线程
5. 复用资源：预分配转换用的字节数组、RenderScript分配器等，减少GC开销

示例：RenderScript加速YUV转RGB

// 初始化RenderScript
private RenderScript _rs;
private ScriptIntrinsicYuvToRGB _yuvToRgbScript;
private Allocation _yuvAllocation;
private Allocation _rgbAllocation;

public FrameAnalyzer(ICameraFrameListener listener)
{
    _listener = listener;
    _rs = RenderScript.Create(Android.App.Application.Context);
    _yuvToRgbScript = ScriptIntrinsicYuvToRGB.Create(_rs, Element.U8_4(_rs));
}

private byte[] YuvToRgbWithRenderScript(IImageProxy image)
{
    int width = image.Width;
    int height = image.Height;

    // 初始化分配器
    if (_yuvAllocation == null || _yuvAllocation.Type.X != width || _yuvAllocation.Type.Y != height)
    {
        Type.Builder yuvType = new Type.Builder(_rs, Element.U8(_rs))
            .SetX(width)
            .SetY(height)
            .SetYuvFormat((int)Android.Graphics.ImageFormatType.Yuv420888);
        _yuvAllocation = Allocation.CreateTyped(_rs, yuvType.Create(), AllocationUsage.Script);

        Type.Builder rgbType = new Type.Builder(_rs, Element.RGBA_8888(_rs))
            .SetX(width)
            .SetY(height);
        _rgbAllocation = Allocation.CreateTyped(_rs, rgbType.Create(), AllocationUsage.Script);
    }

    // 将YUV数据复制到分配器
    _yuvAllocation.CopyFrom(GetYuvByteArray(image));

    // 执行转换
    _yuvToRgbScript.SetInput(_yuvAllocation);
    _yuvToRgbScript.ForEach(_rgbAllocation);

    // 获取RGB数据并转换为RGB24
    byte[] rgbaBuffer = new byte[width * height * 4];
    _rgbAllocation.CopyTo(rgbaBuffer);
    
    byte[] rgb24Buffer = new byte[width * height * 3];
    int rgbIdx = 0;
    for (int i = 0; i < rgbaBuffer.Length; i += 4)
    {
        rgb24Buffer[rgbIdx++] = rgbaBuffer[i];     // R
        rgb24Buffer[rgbIdx++] = rgbaBuffer[i + 1]; // G
        rgb24Buffer[rgbIdx++] = rgbaBuffer[i + 2]; // B
    }

    return rgb24Buffer;
}

// 将IImageProxy的YUV数据合并为单字节数组
private byte[] GetYuvByteArray(IImageProxy image)
{
    var planes = image.GetPlanes();
    int totalSize = planes[0].Buffer.Remaining() + planes[1].Buffer.Remaining() + planes[2].Buffer.Remaining();
    byte[] yuvBytes = new byte[totalSize];
    int offset = 0;

    foreach (var plane in planes)
    {
        plane.Buffer.Get(yuvBytes, offset, plane.Buffer.Remaining());
        offset += plane.Buffer.Remaining();
    }

    return yuvBytes;
}

内容的提问来源于stack exchange，提问作者Kajal