基于MediaPipe Pose Landmarker实现手势触发音频播放的技术咨询

以下是针对需求的完整可运行实现，适配你的HTML/CSS/JavaScript基础，核心功能是检测手部触碰脸部并触发警报：

完整代码 #

HTML #

<!DOCTYPE html>
<html>
<head>
    <title>别摸胡子提醒器</title>
    <link rel="stylesheet" href="styles.css">
    <!-- 引入MediaPipe依赖 -->
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/pose_landmarker/pose_landmarker.js"></script>
</head>
<body>
    <div class="container">
        <video id="webcam" autoplay muted></video>
        <canvas id="output_canvas"></canvas>
        <button id="webcamButton">启用检测</button>
        <!-- 警报音频（可替换成自己的音效） -->
        <audio id="alertSound" src="https://assets.mixkit.co/sfx/preview/mixkit-software-interface-start-2574.mp3" loop></audio>
    </div>
    <script src="app.js"></script>
</body>
</html>

CSS (styles.css) #

.container {
    position: relative;
    width: 640px;
    margin: 20px auto;
}

#webcam, #output_canvas {
    position: absolute;
    width: 100%;
    height: auto;
}

#webcamButton {
    position: absolute;
    bottom: 20px;
    left: 50%;
    transform: translateX(-50%);
    padding: 10px 20px;
    background: #ff4444;
    color: white;
    border: none;
    border-radius: 4px;
    cursor: pointer;
    font-size: 16px;
    z-index: 10;
}

#webcamButton:hover {
    background: #cc0000;
}

JavaScript (app.js) #

const video = document.getElementById("webcam");
const canvasElement = document.getElementById("output_canvas");
const canvasCtx = canvasElement.getContext("2d");
const drawingUtils = new DrawingUtils(canvasCtx);
const alertSound = document.getElementById("alertSound");
let enableWebcamButton;
let webcamRunning = false;
let poseLandmarker;
let runningMode = "IMAGE";
let isTouchingFace = false;

// 初始化姿态检测模型
async function initPoseLandmarker() {
    const vision = await import('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/vision_bundle.js');
    const PoseLandmarker = vision.PoseLandmarker;
    const PoseLandmarkerOptions = vision.PoseLandmarkerOptions;
    const VisionRunningMode = vision.VisionRunningMode;

    poseLandmarker = await PoseLandmarker.createFromOptions(PoseLandmarkerOptions.fromJSON({
        baseOptions: {
            modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
            delegate: "GPU"
        },
        runningMode: VisionRunningMode.IMAGE,
        numPoses: 1
    }));
}

// 检查摄像头支持
const hasGetUserMedia = () => !!navigator.mediaDevices?.getUserMedia;

if (hasGetUserMedia()) {
    enableWebcamButton = document.getElementById("webcamButton");
    enableWebcamButton.addEventListener("click", enableCam);
} else {
    console.warn("浏览器不支持摄像头访问");
}

// 启用/关闭摄像头检测
function enableCam(event) {
    if (!poseLandmarker) {
        console.log("模型加载中，请稍候");
        return;
    }

    webcamRunning = !webcamRunning;
    enableWebcamButton.innerText = webcamRunning ? "关闭检测" : "启用检测";

    if (webcamRunning) {
        const constraints = { video: true };
        navigator.mediaDevices.getUserMedia(constraints).then((stream) => {
            video.srcObject = stream;
            video.addEventListener("loadeddata", predictWebcam);
        });
    } else {
        // 关闭检测时停止音频
        alertSound.pause();
        alertSound.currentTime = 0;
        isTouchingFace = false;
    }
}

// 计算两个关键点的3D距离
function calculateDistance(landmark1, landmark2) {
    const dx = landmark1.x - landmark2.x;
    const dy = landmark1.y - landmark2.y;
    const dz = landmark1.z - landmark2.z;
    return Math.sqrt(dx*dx + dy*dy + dz*dz);
}

// 判断手部是否触碰脸部
function checkHandTouchFace(landmarks) {
    // MediaPipe Pose关键点索引：左右手腕、左右脸颊
    const rightWrist = landmarks[16];
    const leftWrist = landmarks[15];
    const rightCheek = landmarks[24];
    const leftCheek = landmarks[23];

    // 触脸阈值（可根据实际情况调整，值越大越灵敏）
    const touchThreshold = 0.05;

    // 检查左右手是否触碰对应脸颊
    const rightHandTouch = calculateDistance(rightWrist, rightCheek) < touchThreshold;
    const leftHandTouch = calculateDistance(leftWrist, leftCheek) < touchThreshold;

    return rightHandTouch || leftHandTouch;
}

let lastVideoTime = -1;
async function predictWebcam() {
    if (!webcamRunning) return;

    // 同步视频与画布尺寸
    canvasElement.width = video.videoWidth;
    canvasElement.height = video.videoHeight;
    video.style.width = `${video.videoWidth}px`;
    video.style.height = `${video.videoHeight}px`;

    // 切换到视频检测模式
    if (runningMode === "IMAGE") {
        runningMode = "VIDEO";
        await poseLandmarker.setOptions({ runningMode: "VIDEO" });
    }

    const startTimeMs = performance.now();
    if (lastVideoTime !== video.currentTime) {
        lastVideoTime = video.currentTime;
        poseLandmarker.detectForVideo(video, startTimeMs, (result) => {
            canvasCtx.save();
            canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);

            if (result.landmarks.length > 0) {
                const landmarks = result.landmarks[0];
                // 绘制姿态关键点和连接线
                drawingUtils.drawLandmarks(landmarks, {
                    radius: (data) => DrawingUtils.lerp(data.from!.z, -0.15, 0.1, 5, 1)
                });
                drawingUtils.drawConnectors(landmarks, poseLandmarker.POSE_CONNECTIONS);

                // 检测触脸状态并控制音频
                const touchingFace = checkHandTouchFace(landmarks);
                if (touchingFace && !isTouchingFace) {
                    alertSound.play().catch(err => console.log("音频播放失败:", err));
                    isTouchingFace = true;
                } else if (!touchingFace && isTouchingFace) {
                    alertSound.pause();
                    alertSound.currentTime = 0;
                    isTouchingFace = false;
                }
            }

            canvasCtx.restore();
        });
    }

    window.requestAnimationFrame(predictWebcam);
}

// 初始化模型
initPoseLandmarker();

关键功能说明 #

1. 模型初始化：新增initPoseLandmarker函数加载轻量版姿态检测模型，保证浏览器端运行性能。
2. 触脸检测逻辑：通过计算手腕与脸颊关键点的3D距离判断是否触脸，阈值可根据自身情况调整。
3. 音频控制：跟踪触脸状态，仅在状态切换时触发音频播放/暂停，避免重复操作。
4. 浏览器兼容：适配摄像头权限请求，符合浏览器音频自动播放限制（需用户点击按钮触发）。

内容的提问来源于stack exchange，提问作者Garetron