结合Three.js与MediaPipe开发AR手部追踪Web应用的问题及排查

一、先解决当前Three.js报错问题 #

控制台报错TypeError: Cannot read properties of undefined (reading 'center')的根源是：

• initializeThreeJs中的render函数调用updateCubePosition()时未传入landmarks参数，导致函数内部访问landmarks[0]时出现undefined
• predictWebcam中调用的updateCubePosition是initializeThreeJs的内部函数，作用域限制导致外部无法访问

修复步骤： #

1. 将Three.js的核心对象（scene、camera、cube、renderer）提升为全局变量，让predictWebcam可以直接访问并更新cube位置
2. 移除render函数中无参数的updateCubePosition调用，改由predictWebcam在获取手部关键点后更新位置

二、实现Webcam画面作为Three.js场景背景 #

通过将video元素转为Three.js纹理，贴在场景的背景平面上来实现：

• 创建一个覆盖整个相机视口的平面几何体
• 将video作为纹理映射到平面上，放置在相机正前方的合适位置

三、实现手腕关键点绑定3D物体 #

MediaPipe返回的手部关键点是归一化坐标（x、y范围0-1，对应视频宽高比例），需要转换为Three.js的世界坐标：

1. 获取手腕关键点（索引0的关键点）的归一化坐标
2. 将其转换为相机的NDC坐标（范围-1到1）
3. 通过相机的unproject方法转换为世界坐标，确保3D物体始终在相机视野内

完整修复后的代码 #

import {
  HandLandmarker,
  FilesetResolver
} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
import * as THREE from 'https://cdn.skypack.dev/three@0.132.2';

const video = document.getElementById("webcam");
const canvasElement = document.getElementById("output_canvas");
const canvasCtx = canvasElement.getContext("2d");
const cubeContainer = document.getElementById("cube-container");

let results = {};
let handLandmarker;
let webcamRunning = false;

// Three.js全局对象，让predictWebcam可以访问
let scene, camera, renderer, cube;

const enableWebcamButton = document.getElementById("webcamButton");
enableWebcamButton.addEventListener("click", toggleWebcam);

const initializeHandLandmarker = async () => {
  const vision = await FilesetResolver.forVisionTasks(
    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"
  );
  handLandmarker = await HandLandmarker.createFromOptions(vision, {
    baseOptions: {
      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
      delegate: "GPU"
    },
    runningMode: "LIVE",
    numHands: 1
  });
};
initializeHandLandmarker();

async function toggleWebcam() {
  if (!handLandmarker) {
    console.log("Please wait for HandLandmarker to load.");
    return;
  }

  webcamRunning = !webcamRunning;

  if (webcamRunning) {
    enableWebcamButton.innerText = "DISABLE PREDICTIONS";
    await startWebcam();
    initializeThreeJs();
    predictWebcam();
  } else {
    enableWebcamButton.innerText = "ENABLE PREDICTIONS";
    stopWebcam();
  }
}

async function startWebcam() {
  const stream = await navigator.mediaDevices.getUserMedia({ video: true });
  video.srcObject = stream;
  await video.play();

  const { videoWidth, videoHeight } = video;
  canvasElement.width = videoWidth;
  canvasElement.height = videoHeight;

  cubeContainer.style.width = video.videoWidth + 'px';
  cubeContainer.style.height = video.videoHeight + 'px';
}

function stopWebcam() {
  const stream = video.srcObject;
  const tracks = stream.getTracks();
  tracks.forEach(track => track.stop());
  video.srcObject = null;
}

async function predictWebcam() {
  if (!webcamRunning) return;

  if (!handLandmarker) {
    console.log("Please wait for HandLandmarker to load.");
    return;
  }

  const startTimeMs = performance.now();
  results = await handLandmarker.detectForVideo(video, startTimeMs);

  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
  if (results.landmarks) {
    for (const landmarks of results.landmarks) {
      drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
        color: "#00FF00",
        lineWidth: 5
      });
      drawLandmarks(
        canvasCtx,
        landmarks,
        { color: "#FF0000", lineWidth: 2 },
        video.offsetWidth,
        video.offsetHeight
      );

      // 直接更新cube位置
      updateCubePosition(landmarks);
    }
  }

  requestAnimationFrame(predictWebcam);
}

//________Three.js_________

function initializeThreeJs() {
  scene = new THREE.Scene();

  // 创建透视相机
  camera = new THREE.PerspectiveCamera(75, video.videoWidth / video.videoHeight, 0.1, 1000);
  camera.position.z = 2;

  // 创建WebGL渲染器
  renderer = new THREE.WebGLRenderer({alpha: true});
  renderer.setSize(video.videoWidth, video.videoHeight);
  renderer.domElement.style.position = 'absolute';
  renderer.domElement.style.top = '0';
  renderer.domElement.style.left = '0';
  cubeContainer.appendChild(renderer.domElement);

  // 1. 实现Webcam作为背景
  const videoTexture = new THREE.VideoTexture(video);
  videoTexture.minFilter = THREE.LinearFilter;
  videoTexture.magFilter = THREE.LinearFilter;
  // 创建背景平面，覆盖整个视口
  const bgGeometry = new THREE.PlaneGeometry(
    2 * camera.aspect * Math.tan(camera.fov / 2 * Math.PI / 180) * camera.position.z,
    2 * Math.tan(camera.fov / 2 * Math.PI / 180) * camera.position.z
  );
  const bgMaterial = new THREE.MeshBasicMaterial({ map: videoTexture, side: THREE.DoubleSide });
  const bgPlane = new THREE.Mesh(bgGeometry, bgMaterial);
  bgPlane.position.z = camera.position.z - 0.1; // 放在相机前方，比cube近一点
  scene.add(bgPlane);

  // 2. 创建3D立方体
  const cubeMaterial = new THREE.MeshBasicMaterial({ color: 0x00ff00 });
  const cubeSize = 0.1;
  const cubeGeometry = new THREE.BoxGeometry(cubeSize, cubeSize, cubeSize);
  cube = new THREE.Mesh(cubeGeometry, cubeMaterial);
  scene.add(cube);

  // 渲染函数只负责渲染场景，位置更新交给predictWebcam
  function render() {
    renderer.render(scene, camera);
    requestAnimationFrame(render);
  }

  render();
}

// 全局的updateCubePosition函数，处理手腕坐标转换
function updateCubePosition(landmarks) {
  if (!cube || !landmarks || landmarks.length === 0) return;

  // 获取手腕关键点（MediaPipe索引0为手腕）
  const wrist = landmarks[0];
  // 归一化坐标转NDC坐标（x:0-1 → -1到1，y:0-1 → 1到-1，因为Three.js的y轴向上）
  const ndcX = (wrist.x * 2) - 1;
  const ndcY = -(wrist.y * 2) + 1;

  // 将NDC坐标转换为世界坐标
  const vector = new THREE.Vector3(ndcX, ndcY, 0.5);
  vector.unproject(camera);

  // 计算方向向量，确保cube在相机前方固定距离
  const dir = vector.sub(camera.position).normalize();
  const distance = camera.position.z - 0.5; // 距离相机的距离，可调整
  cube.position.copy(camera.position.clone().add(dir.multiplyScalar(distance)));
}

关键修改说明： #

1. 报错修复：将Three.js核心对象设为全局，移除render函数中无参数的updateCubePosition调用，改由predictWebcam传入landmarks更新位置
2. Webcam背景实现：创建VideoTexture并映射到背景平面，根据相机参数计算平面大小，确保覆盖整个视口
3. 手腕坐标绑定：使用unproject方法将归一化坐标转换为世界坐标，确保cube始终在相机视野内，位置与手腕精准对应

内容的提问来源于stack exchange，提问作者Pigi