MediaPipe手语翻译器双手识别特征不匹配问题求助

使用MediaPipe开发实时手语翻译器，单手识别时运行正常，但举起双手时触发错误：

X has 84 features, but RandomForestClassifier is expecting 42 features as input

运行代码如下：

import pickle
import cv2
import mediapipe as mp
import numpy as np

model_dict = pickle.load(open('./model.p', 'rb'))
model = model_dict['model']

cap = cv2.VideoCapture(0)

mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles

hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)

labels_dict = {0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'J', 10: 'K', 11: 'L',
               12: 'M', 13: 'N', 14: 'O', 15: 'P', 16: 'Q', 17: 'R', 18: 'S', 19: 'T', 20: 'U', 21: 'V', 22: 'W',
               23: 'X', 24: 'Y', 25: 'Z', 26: 'Middelvinger', 27: 'Dikke vette duim'}

current_char = "niks"  # Initialiseer current_char buiten de loop

while True:
    try:
        # Initialiseer lege lijsten en coördinaten
        data_aux = []
        x_ = []
        y_ = []

        # Lees een frame van de webcam
        ret, frame = cap.read()

        # Haal de hoogte, breedte en kanalen van het frame op
        H, W, _ = frame.shape

        # Converteer het frame naar RGB-formaat
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

        # Verwerk de handen in het frame met behulp van MediaPipe Hands
        results = hands.process(frame_rgb)

        # Controleer of handen zijn gedetecteerd in het frame
        if results.multi_hand_landmarks:
            # Loop over alle gedetecteerde handen in het frame
            for hand_landmarks in results.multi_hand_landmarks:
                # Teken de landmarks en verbindingslijnen op het frame
                mp_drawing.draw_landmarks(
                    frame,  # afbeelding om op te tekenen
                    hand_landmarks,  # model output
                    mp_hands.HAND_CONNECTIONS,  # verbindingslijnen
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())

            # Loop over alle gedetecteerde handen in het frame
            for hand_landmarks in results.multi_hand_landmarks:
                # Loop over alle landmarks (puntcoördinaten) van de hand
                for i in range(len(hand_landmarks.landmark)):
                    x = hand_landmarks.landmark[i].x
                    y = hand_landmarks.landmark[i].y

                    # Voeg de x- en y-coördinaten toe aan de respectievelijke lijsten
                    x_.append(x)
                    y_.append(y)

                # Normaliseer de coördinaten door de minimale x- en y-waarden af te trekken
                for i in range(len(hand_landmarks.landmark)):
                    x = hand_landmarks.landmark[i].x
                    y = hand_landmarks.landmark[i].y
                    data_aux.append(x - min(x_))
                    data_aux.append(y - min(y_))

            # Bereken de positie om een tekstvenster te plaatsen
            x1 = int(min(x_) * W) - 10
            y1 = int(min(y_) * H) - 10

            # Voorspel het karakter met behulp van het getrainde model
            prediction = model.predict([np.asarray(data_aux)])
            predicted_character = labels_dict[int(prediction[0])]

            # Haal de waarschijnlijkheid van de voorspelling op
            prob = model.predict_proba([np.asarray(data_aux)])
            accuracy_percentage = max(prob[0]) * 100

            # Toon het voorspelde karakter en de nauwkeurigheid in de linkerbovenhoek
            cv2.putText(frame, f"{predicted_character} {accuracy_percentage:.2f}%", (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 3, cv2.LINE_AA)

            # Controleer of het karakter is veranderd
            if predicted_character != current_char:
                print(f"{accuracy_percentage:.2f}% zekerheid dat het {predicted_character} is")
                current_char = predicted_character

        # Toon het frame met de getekende handen en voorspellingen
        cv2.imshow('frame', frame)
        cv2.waitKey(1)

    except Exception as e:
        print(f"Fout code: {e}")

cap.release()
cv2.destroyAllWindows()

期望程序能够同时扫描并翻译双手动作。

核心原因 #

你的RandomForestClassifier是用单只手的42维特征（21个关键点×x/y坐标）训练的，但双手时代码会把两只手的特征合并成84维，导致特征维度不匹配，触发报错。

具体解决方案 #

方案1：单独处理每只手（无需重新训练模型）

保留现有单手势模型，对每只手单独提取特征、预测，这样每只手的特征维度都符合模型要求：

• 修改代码中特征提取和预测的逻辑，遍历每只手时，单独初始化该手的坐标列表和特征列表，不要共用全局的x_、y_、data_aux
• 用单只手自身的最小坐标做归一化，而不是全局的最小坐标
• 每只手单独预测，并在对应位置标注结果

修改后的关键代码片段：

# 替换原有的双手遍历预测逻辑
if results.multi_hand_landmarks:
    # 先绘制所有手的关键点
    for hand_landmarks in results.multi_hand_landmarks:
        mp_drawing.draw_landmarks(
            frame,
            hand_landmarks,
            mp_hands.HAND_CONNECTIONS,
            mp_drawing_styles.get_default_hand_landmarks_style(),
            mp_drawing_styles.get_default_hand_connections_style())
    
    # 遍历每只手单独处理
    for hand_idx, hand_landmarks in enumerate(results.multi_hand_landmarks):
        x_single = []
        y_single = []
        data_aux_single = []
        
        # 收集当前手的所有关键点坐标
        for i in range(len(hand_landmarks.landmark)):
            x = hand_landmarks.landmark[i].x
            y = hand_landmarks.landmark[i].y
            x_single.append(x)
            y_single.append(y)
        
        # 用当前手的最小坐标做归一化
        for i in range(len(hand_landmarks.landmark)):
            x = hand_landmarks.landmark[i].x
            y = hand_landmarks.landmark[i].y
            data_aux_single.append(x - min(x_single))
            data_aux_single.append(y - min(y_single))
        
        # 单只手预测
        prediction = model.predict([np.asarray(data_aux_single)])
        predicted_character = labels_dict[int(prediction[0])]
        prob = model.predict_proba([np.asarray(data_aux_single)])
        accuracy_percentage = max(prob[0]) * 100
        
        # 在当前手的上方标注结果
        x_min = min(x_single) * W
        y_min = min(y_single) * H - 30
        cv2.putText(frame, f"手{hand_idx+1}: {predicted_character} {accuracy_percentage:.2f}%", 
                    (int(x_min), int(y_min)), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2, cv2.LINE_AA)
        
        # 日志输出
        if predicted_character != current_char:
            print(f"手{hand_idx+1}: {accuracy_percentage:.2f}% 确信是 {predicted_character}")
            current_char = predicted_character

方案2：支持双手组合手势（需重新训练模型）

如果需要识别双手组合的特定手语，需要：

• 重新收集数据集：每个样本包含两只手的84维特征（2×21×2），对应双手组合的手势标签
• 用新的数据集重新训练RandomForestClassifier
• 修改代码，将两只手的特征合并成84维后输入模型预测

内容的提问来源于stack exchange，提问作者mvw007