如何消除手部检测控制鼠标光标移动时的抖动问题？

问题背景 #

我正在开发一个Python计算机视觉项目：用OpenCV捕获摄像头画面，通过MediaPipe检测手部，借助AutoPy实现鼠标跟随食指移动。但遇到核心问题：手部静止时，鼠标光标仍会剧烈抖动、不受控移动。原项目脚本如下：

import cv2
import autopy
import mediapipe as mp
cap = cv2.VideoCapture(0)
width, height = autopy.screen.size()
hands = mp.solutions.hands.Hands(static_image_mode=False, max_num_hands=1, min_tracking_confidence=0.5,
                                 min_detection_confidence=0.5)
mpDraw = mp.solutions.drawing_utils
f1, f2 = False, False
while True:
    _, img = cap.read()
    img = cv2.flip(img, 1)
    result = hands.process(img)
    if f1 and not f2:
        print('\nHand disappeared')
    if result.multi_hand_landmarks:
        for id_finger, lm in enumerate(result.multi_hand_landmarks[0].landmark):
            h, w, _ = img.shape
            f1 = f2
            f2 = True
            if not f1 and f2:
                print('Hand appeared')
            cx, cy = int(lm.x * w), int(lm.y * h)
            cv2.circle(img, (cx, cy), 3, (255, 0, 255))
            if id_finger == 4:
                cx_2, cy_2 = cx, cy
            if id_finger == 8:
                cv2.circle(img, (cx, cy), 25, (255, 0, 255), cv2.FILLED)
                cx_1, cy_1 = cx, cy
                try:
                    autopy.mouse.move(cx * width / w, cy * height / h)
                    print(cx, cy, sep=' ', end='; ')
                except ValueError:
                    continue
                if ((cx_1 - cx_2) ** 2 + (cy_1 - cy_2) ** 2) ** 0.5 < 50:
                    try:
                        autopy.mouse.click()
                    except ValueError:
                        continue
        mpDraw.draw_landmarks(img, result.multi_hand_landmarks[0], mp.solutions.hands.HAND_CONNECTIONS)
    else:
        f1 = f2
        f2 = False
    cv2.imshow("Hand tracking", img)
    cv2.waitKey(1)

解决抖动的核心方法 #

1. 指数平滑过滤坐标波动 #

对连续帧的食指坐标做加权平均，给历史坐标更高权重、当前坐标较低权重，平衡响应速度与平滑度，抵消单帧检测的微小波动。

2. 设置移动阈值 #

仅当当前检测的食指坐标与上一次鼠标位置的距离超过设定阈值时，才更新鼠标位置，忽略检测误差带来的微小偏移。

3. 提高MediaPipe置信度 #

调高min_tracking_confidence和min_detection_confidence参数，过滤低置信度的检测结果，减少无效的坐标跳变。

4. 限制鼠标最大移动步长 #

强制限制单帧内鼠标的最大移动距离，避免因检测跳变导致鼠标瞬移。

修改后的完整代码 #

import cv2
import autopy
import mediapipe as mp
import numpy as np

cap = cv2.VideoCapture(0)
screen_width, screen_height = autopy.screen.size()

# 提高MediaPipe置信度，过滤低质量检测结果
hands = mp.solutions.hands.Hands(
    static_image_mode=False,
    max_num_hands=1,
    min_tracking_confidence=0.7,
    min_detection_confidence=0.7
)
mpDraw = mp.solutions.drawing_utils

# 平滑相关参数
prev_mouse_x, prev_mouse_y = 0, 0
smoothing_factor = 0.2  # 0-1，越小鼠标越平滑但响应稍慢
move_threshold = 5      # 触发鼠标移动的最小像素距离
max_step = 20           # 单帧鼠标最大移动步长

f1, f2 = False, False
while True:
    _, img = cap.read()
    img = cv2.flip(img, 1)
    img_h, img_w, _ = img.shape
    result = hands.process(img)

    if f1 and not f2:
        print('\nHand disappeared')
    if result.multi_hand_landmarks:
        hand_landmarks = result.multi_hand_landmarks[0]
        mpDraw.draw_landmarks(img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS)
        
        f1 = f2
        f2 = True
        if not f1 and f2:
            print('Hand appeared')

        # 获取食指和拇指指尖坐标
        index_finger = hand_landmarks.landmark[8]
        thumb_finger = hand_landmarks.landmark[4]

        # 转换为图像像素坐标
        cx, cy = int(index_finger.x * img_w), int(index_finger.y * img_h)
        cx_2, cy_2 = int(thumb_finger.x * img_w), int(thumb_finger.y * img_h)

        cv2.circle(img, (cx, cy), 25, (255, 0, 255), cv2.FILLED)

        # 转换为屏幕坐标
        target_x = cx * screen_width / img_w
        target_y = cy * screen_height / img_h

        # 应用指数平滑
        current_mouse_x = prev_mouse_x * (1 - smoothing_factor) + target_x * smoothing_factor
        current_mouse_y = prev_mouse_y * (1 - smoothing_factor) + target_y * smoothing_factor

        # 距离判断，超过阈值才移动鼠标
        distance = np.sqrt((current_mouse_x - prev_mouse_x)**2 + (current_mouse_y - prev_mouse_y)**2)
        if distance > move_threshold:
            # 限制最大移动步长
            step_x = current_mouse_x - prev_mouse_x
            step_y = current_mouse_y - prev_mouse_y
            step_length = np.sqrt(step_x**2 + step_y**2)
            if step_length > max_step:
                scale = max_step / step_length
                step_x *= scale
                step_y *= scale
                current_mouse_x = prev_mouse_x + step_x
                current_mouse_y = prev_mouse_y + step_y

            try:
                autopy.mouse.move(current_mouse_x, current_mouse_y)
                prev_mouse_x, prev_mouse_y = current_mouse_x, current_mouse_y
                print(int(current_mouse_x), int(current_mouse_y), sep=' ', end='; ')
            except ValueError:
                continue

        # 保留原点击判断逻辑
        if np.sqrt((cx - cx_2)**2 + (cy - cy_2)**2) < 50:
            try:
                autopy.mouse.click()
            except ValueError:
                continue
    else:
        f1 = f2
        f2 = False

    cv2.imshow("Hand tracking", img)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

代码说明 #

• 指数平滑：通过smoothing_factor参数调整平滑程度，可根据实际需求在0.1-0.5之间取值。
• 移动阈值：move_threshold过滤微小检测波动，建议设置为3-8像素。
• 置信度调整：将置信度从0.5提高到0.7，有效减少低质量检测带来的坐标跳变。
• 步长限制：max_step避免鼠标因检测误差瞬间移动过大，建议设置为15-25像素。

内容的提问来源于stack exchange，提问作者Тимофей Якушев