QVideoFilterRunnable修改无显示：MediaPipe人脸检测QML滤镜问题

问题背景 #

开发基于MediaPipe的人脸检测视频滤镜，采用QML实现多摄像头适配，但QVideoFilterRunnable处理后的画面无法在VideoOutput组件中显示，相关代码如下：

QML代码 #

import QtQuick 2.15
import QtQuick.Controls 2.15
import QtMultimedia 5.15
import QtQuick.Layouts 1.15
import CustomFilter 1.0

Item {
    width:1280
    height:720

    Camera {
        id: camera
    }

    FaceDetectionFilter {
        id: myFaceDetectionFilter
    }

    VideoOutput {
        id: viewfinder
        filters: [ myFaceDetectionFilter ]
        source: camera
        anchors.fill: parent
    }
}

Python滤镜代码 #

mp_face_detection = mp.solutions.face_detection
mp_drawing = mp.solutions.drawing_utils

class FaceDetectionFilterRunnable(QVideoFilterRunnable):
    def __init__(self, parent=None):
        super().__init__(parent)

    def run(self, _input, surface, flags):
        return self.transform(_input)
    
    def processFrame(self, frame: QVideoFrame) -> QImage:
        img = frame.image().convertToFormat(QImage.Format_RGB888)
        img = self.QImageToCvMat(img)
        img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)

        with mp_face_detection.FaceDetection( model_selection=0, min_detection_confidence=0.5) as face_detection:
            img=cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            results = face_detection.process(img)
            if results.detections:
                for detection in results.detections:
                    mp_drawing.draw_detection(img, detection)

        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        height, width, _ = img.shape

        bytesPerLine = 3 * width
        qImg = QImage(img.data, width, height, bytesPerLine, QImage.Format_RGB888)
        return qImg

    def filterAcceptsVideoFrame(self, frame: QVideoFrame) -> bool:
        return True

    def transform(self, frame: QVideoFrame) -> QVideoFrame:
        img = self.processFrame(frame)
        
        if not img.isNull():
            return QVideoFrame(img)
        return frame
    def QImageToCvMat(self,incomingImage):
        '''  Converts a QImage into an opencv MAT format  '''
        incomingImage = incomingImage.convertToFormat(QImage.Format_RGB32)
        width = incomingImage.width()
        height = incomingImage.height()
        ptr = incomingImage.constBits()
        arr = np.array(ptr).reshape(height, width, 4)  #  Copies the data
        return arr

class FaceDetectionFilter(QAbstractVideoFilter):
    def createFilterRunnable(self):
        return FaceDetectionFilterRunnable(self)

注册代码 #

qmlRegisterType(FaceDetectionFilter, "CustomFilter", 1, 0, "FaceDetectionFilter")

核心问题分析 #

1. 内存生命周期不匹配：通过QImage(img.data...)创建的QImage未持有数据内存所有权，numpy数组img在方法执行完后会被回收，导致QVideoFrame引用无效内存，画面无法渲染。
2. 颜色空间转换冗余且错误：多次来回转换RGB/BGR，既降低性能又容易导致颜色通道混乱，最终画面显示异常。
3. QImage转OpenCV Mat格式错误：将QImage转为Format_RGB32（实际是BGRA四通道）后直接转为三通道数组，通道信息混乱。

修复后的代码 #

修正后的Python滤镜代码 #

mp_face_detection = mp.solutions.face_detection
mp_drawing = mp.solutions.drawing_utils
import numpy as np
import cv2
from PyQt5.QtMultimedia import QVideoFilterRunnable, QAbstractVideoFilter, QVideoFrame
from PyQt5.QtGui import QImage

class FaceDetectionFilterRunnable(QVideoFilterRunnable):
    def __init__(self, parent=None):
        super().__init__(parent)
        # 初始化MediaPipe检测器，避免每次帧都创建，提升性能
        self.face_detection = mp_face_detection.FaceDetection(model_selection=0, min_detection_confidence=0.5)

    def run(self, _input, surface, flags):
        return self.transform(_input)
    
    def processFrame(self, frame: QVideoFrame) -> QImage:
        # 从QVideoFrame获取QImage，直接用RGB888格式
        q_img = frame.image().convertToFormat(QImage.Format_RGB888)
        # 转换为OpenCV的RGB格式Mat
        cv_img = self.QImageToCvMat(q_img)

        # MediaPipe处理RGB格式图像
        results = self.face_detection.process(cv_img)
        if results.detections:
            for detection in results.detections:
                mp_drawing.draw_detection(cv_img, detection)

        # 将处理后的RGB Mat转回QImage，必须深拷贝确保内存有效
        height, width = cv_img.shape[:2]
        bytes_per_line = 3 * width
        # 使用copy()让QImage持有独立内存
        return QImage(cv_img.data, width, height, bytes_per_line, QImage.Format_RGB888).copy()

    def filterAcceptsVideoFrame(self, frame: QVideoFrame) -> bool:
        return frame.isValid()

    def transform(self, frame: QVideoFrame) -> QVideoFrame:
        if not frame.isValid():
            return frame
        processed_img = self.processFrame(frame)
        if not processed_img.isNull():
            return QVideoFrame(processed_img)
        return frame

    def QImageToCvMat(self, incomingImage: QImage) -> np.ndarray:
        ''' 正确转换QImage(Format_RGB888)到OpenCV RGB Mat '''
        width = incomingImage.width()
        height = incomingImage.height()
        # 获取QImage的原始数据指针
        ptr = incomingImage.constBits()
        # 直接转为三通道RGB数组
        arr = np.array(ptr).reshape(height, width, 3)
        return arr

    def __del__(self):
        # 释放MediaPipe检测器资源
        self.face_detection.close()

class FaceDetectionFilter(QAbstractVideoFilter):
    def createFilterRunnable(self):
        return FaceDetectionFilterRunnable(self)

QML代码优化（添加摄像头激活与权限检查） #

import QtQuick 2.15
import QtQuick.Controls 2.15
import QtMultimedia 5.15
import QtQuick.Layouts 1.15
import CustomFilter 1.0

Item {
    width:1280
    height:720

    Camera {
        id: camera
        active: true // 主动激活摄像头
        onAuthorizationStatusChanged: {
            if (authorizationStatus === Camera.Denied) {
                console.log("摄像头权限被拒绝")
            }
        }
    }

    FaceDetectionFilter {
        id: myFaceDetectionFilter
    }

    VideoOutput {
        id: viewfinder
        filters: [ myFaceDetectionFilter ]
        source: camera
        anchors.fill: parent
        visible: camera.active // 仅当摄像头激活时显示
    }
}

关键修复点说明 #

• 内存管理：通过QImage.copy()深拷贝数据，确保QVideoFrame使用的内存不会被numpy回收。
• 颜色通道简化：全程使用RGB格式，避免多余的BGR/RGB转换，减少出错概率。
• 性能优化：将MediaPipe检测器初始化移到__init__方法，避免每帧创建销毁，提升处理速度。
• 有效性检查：添加帧有效性判断，避免处理无效帧导致崩溃。

内容的提问来源于stack exchange，提问作者Enis Yalçın