Flutter中CameraImage/Uint8List转RGB数组对接Python姿态估计后端

一、依赖准备 #

使用官方维护的image库替代已废弃的Imagi库处理图像转换，在pubspec.yaml中添加依赖：

dependencies:
  image: ^4.0.17 # 建议使用最新稳定版本

二、处理跨平台CameraImage格式差异 #

CameraImage在Android上多为YUV_420_888格式，iOS上常为BGRA格式，需分别处理：

1. Android YUV_420_888 转 RGB #

import 'package:image/image.dart' as img;

img.Image convertYUV420ToImage(CameraImage cameraImage) {
  final int width = cameraImage.width;
  final int height = cameraImage.height;

  final int uvRowStride = cameraImage.planes[1].bytesPerRow;
  final int uvPixelStride = cameraImage.planes[1].bytesPerPixel!;

  final img.Image image = img.Image(width, height);

  for (int w = 0; w < width; w++) {
    for (int h = 0; h < height; h++) {
      final int uvIndex = uvPixelStride * (w ~/ 2) + uvRowStride * (h ~/ 2);
      final int index = h * width + w;

      final y = cameraImage.planes[0].bytes[index];
      final u = cameraImage.planes[1].bytes[uvIndex];
      final v = cameraImage.planes[2].bytes[uvIndex];

      // YUV转RGB公式，确保值在0-255范围内
      int r = (y + 1.370705 * (v - 128)).round().clamp(0, 255);
      int g = (y - 0.698001 * (v - 128) - 0.337633 * (u - 128)).round().clamp(0, 255);
      int b = (y + 1.732446 * (u - 128)).round().clamp(0, 255);

      image.data[index] = (0xFF << 24) | (r << 16) | (g << 8) | b;
    }
  }

  return image;
}

2. iOS BGRA 转 RGB #

import 'package:image/image.dart' as img;

img.Image convertBGRAToImage(CameraImage cameraImage) {
  final int width = cameraImage.width;
  final int height = cameraImage.height;
  final bytes = cameraImage.planes[0].bytes;
  
  return img.Image.fromBytes(width, height, bytes, format: img.Format.bgra);
}

三、图像流中转换并发送到后端 #

在相机流中判断平台完成转换，直接以字节流发送，避免转PNG的性能损耗：

import 'dart:io';
import 'package:http/http.dart' as http;

controller!.startImageStream((CameraImage img) async {
  if (!isDetecting) {
    isDetecting = true;
    img.Image? convertedImage;

    if (Platform.isAndroid) {
      convertedImage = convertYUV420ToImage(img);
    } else if (Platform.isIOS) {
      convertedImage = convertBGRAToImage(img);
    }

    if (convertedImage == null) {
      isDetecting = false;
      return;
    }

    // 提取RGB格式字节流（R-G-B顺序）
    Uint8List rgbBytes = Uint8List.fromList(convertedImage.getBytes(format: img.Format.rgb));

    // 发送到后端API
    try {
      final response = await http.post(
        Uri.parse('http://你的后端地址/predict'),
        body: rgbBytes,
        headers: {'Content-Type': 'application/octet-stream'},
      );
      // 根据后端返回处理结果
      if (response.statusCode == 200) {
        // 处理姿态估计结果
      }
    } catch (e) {
      print('请求异常: $e');
    }

    isDetecting = false;
  }
});

四、后端接收并转成numpy数组 #

Python后端直接接收字节流，转换为numpy数组后送入姿态估计模型：

import numpy as np
import cv2
import mediapipe as mp
from flask import Flask, request

app = Flask(__name__)
mp_pose = mp.solutions.pose

@app.route('/predict', methods=['POST'])
def predict():
    # 假设前端固定相机分辨率为640x480，可根据实际调整或从请求头传递尺寸
    width = 640
    height = 480
    rgb_bytes = request.get_data()
    
    # 字节流转numpy RGB数组
    image_rgb = np.frombuffer(rgb_bytes, dtype=np.uint8).reshape(height, width, 3)
    # 转为模型需要的BGR格式
    image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)

    with mp_pose.Pose(min_detection_confidence=0.5, min_tracking_confidence=0.5) as pose:
        image_bgr.flags.writeable = False
        results = pose.process(image_bgr)
        # 处理姿态估计结果并返回
        return {'status': 'success', 'landmarks': str(results.pose_landmarks)}

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)

注意事项 #

• 固定相机分辨率：前端固定相机输出尺寸，后端提前知晓可避免reshape错误；如需动态尺寸，可在请求头中传递width/height参数。
• 性能优化：Flutter端可使用Isolate处理图像转换，避免阻塞UI线程；后端采用异步框架提升并发处理能力。
• 格式兼容：部分设备可能存在特殊格式，可通过cameraImage.format.group和cameraImage.format.raw字段判断并适配。

内容的提问来源于stack exchange，提问作者dracule22