如何使用WebCodecs VideoDecoder回退帧时避免渲染中间delta帧

我用WebCodecs VideoDecoder和mp4box.js实现了支持逐帧控制的自定义MP4视频播放器，但受VideoDecoder限制，回退到前一帧时必须处理最近关键帧到目标帧之间的所有帧，导致这些中间帧都会渲染到画布，观感很差。请问怎么在回退帧时只渲染目标帧，避免显示中间帧？

我的代码 #

<!DOCTYPE html>
<html lang="en">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Custom Video Player</title>
</head>

<body>
  <canvas id="videoCanvas" width="640" height="360"></canvas>
  <br>
  <input type="file" id="fileInput" accept="video/mp4">
  <button id="play">Play</button>
  <button id="pause">Pause</button>
  <button id="nextFrame">Next frame</button>
  <button id="prevFrame">Previous frame</button>

  <script src="mp4box.all.min.js"></script>
  <script>
    const fileInput = document.getElementById('fileInput');
    const playButton = document.getElementById('play');
    const pauseButton = document.getElementById('pause');
    const nextFrameButton = document.getElementById('nextFrame');
    const prevFrameButton = document.getElementById('prevFrame');
    const canvas = document.getElementById('videoCanvas');
    const ctx = canvas.getContext('2d');

    let mp4boxFile;
    let videoDecoder;
    let playing = false;
    let frameDuration = 1000 / 50; // 50 fps
    let currentFrame = 0;
    let frames = [];
    let shouldRenderFrame = true;


    function findPreviousKeyFrame(frameIndex) {
      for (let i = frameIndex - 1; i >= 0; i--) {
        if (frames[i].type === 'key') {
          return i;
        }
      }
      return -1;
    }

    async function displayFramesInRange(start, end) {
      shouldRenderFrame = false;
      for (let i = start; i < end; i++) {
        if (i == end - 1) {
          shouldRenderFrame = true;
          console.log("end");
        }
        await videoDecoder.decode(frames[i]);
      }
    }

    function shouldRenderNextFrame() {
      return shouldRenderFrame;
    }

    async function prevFrame() {
      if (playing || currentFrame <= 1) return;

      // Find the previous keyframe.
      const keyFrameIndex = findPreviousKeyFrame(currentFrame - 1);

      // If no keyframe found, we can't go back.
      if (keyFrameIndex === -1) return;

      // Display frames from the previous keyframe up to the desired frame.
      await displayFramesInRange(keyFrameIndex, currentFrame - 1);
      currentFrame--;
    }

    async function initVideoDecoder() {
      videoDecoder = new VideoDecoder({
        output: displayFrame,
        error: e => console.error(e),
      });
    }

    function displayFrame(frame) {
      if (shouldRenderNextFrame()) {
        ctx.drawImage(frame, 0, 0);
      }
      frame.close();
    }

    function playVideo() {
      if (playing) return;
      console.log('Playing video');
      playing = true;
      (async () => {
        for (let i = currentFrame; i < frames.length && playing; i++) {
          await videoDecoder.decode(frames[i]);
          currentFrame = i + 1;
          await new Promise(r => setTimeout(r, frameDuration));
        }
        playing = false;
      })();
    }

    function getDescription(trak) {
      for (const entry of trak.mdia.minf.stbl.stsd.entries) {
        if (entry.avcC || entry.hvcC) {
          const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
          if (entry.avcC) {
            entry.avcC.write(stream);
          } else {
            entry.hvcC.write(stream);
          }
          return new Uint8Array(stream.buffer, 8);  // Remove the box header.
        }
      }
      throw "avcC or hvcC not found";
    }

    function pauseVideo() {
      playing = false;
    }

    function nextFrame() {
      if (playing || currentFrame >= frames.length) return;
      videoDecoder.decode(frames[currentFrame]);
      currentFrame++;
    }

    fileInput.addEventListener('change', () => {
      if (!fileInput.files[0]) return;
      const fileReader = new FileReader();
      fileReader.onload = e => {
        mp4boxFile = MP4Box.createFile();
        mp4boxFile.onReady = info => {
          const videoTrack = info.tracks.find(track => track.type === 'video');
          const trak = mp4boxFile.getTrackById(videoTrack.id);
          videoDecoder.configure({
            codec: videoTrack.codec,
            codedHeight: videoTrack.video.height,
            codedWidth: videoTrack.video.width,
            description: this.getDescription(trak)
          });
          mp4boxFile.setExtractionOptions(videoTrack.id);
          mp4boxFile.start()
          mp4boxFile.onSamples = (id, user, samples) => {
            frames.push(...samples.map(sample => new EncodedVideoChunk({
              type: sample.is_sync
                ? 'key' : 'delta',
              timestamp: sample.dts,
              data: sample.data.buffer,
            })));
          };
          mp4boxFile.flush();
        };
        e.target.result.fileStart = 0;
        mp4boxFile.appendBuffer(e.target.result);
      };
      fileReader.readAsArrayBuffer(fileInput.files[0]);
    });

    playButton.addEventListener('click', playVideo);
    pauseButton.addEventListener('click', pauseVideo);
    nextFrameButton.addEventListener('click', nextFrame);
    prevFrameButton.addEventListener('click', prevFrame);

    initVideoDecoder();

  </script>
</body>

</html>

解决方案 #

核心思路是解码中间帧时不更新画布，仅在解码到目标帧时才渲染，同时解决异步解码的时序问题，避免简单变量开关失效。

1. 替换渲染控制变量，跟踪目标帧索引 #

把原有的shouldRenderFrame布尔变量改成记录目标帧索引的变量，避免异步操作导致的时序错误：

let targetRenderFrameIndex = null; // 仅渲染此索引对应的帧

2. 修改displayFramesInRange，标记需要渲染的目标帧 #

在解码序列中明确标记最后一帧为目标渲染帧，并给每个编码帧附加索引，方便后续识别：

async function displayFramesInRange(start, end) {
  targetRenderFrameIndex = end - 1; // 只有最后一帧需要渲染
  for (let i = start; i < end; i++) {
    // 给编码帧附加索引，后续解码时识别
    const chunkWithIndex = {...frames[i], _index: i};
    await videoDecoder.decode(chunkWithIndex);
  }
}

3. 更新displayFrame函数，仅渲染目标帧 #

解码输出时，检查当前帧是否为目标帧，是才渲染到画布：

function displayFrame(frame, metadata) {
  // 从解码元数据中获取帧索引
  const frameIndex = metadata?.decodedChunk?._index;
  if (frameIndex !== undefined && frameIndex === targetRenderFrameIndex) {
    ctx.drawImage(frame, 0, 0);
    targetRenderFrameIndex = null; // 渲染完成后重置标记
  }
  frame.close();
}

注意：如果浏览器不支持通过metadata传递自定义属性，可以用Map存储帧索引与timestamp的映射（确保timestamp唯一），通过timestamp匹配索引。

4. 修正prevFrame的帧计数逻辑 #

确保回退后的currentFrame正确对应目标帧：

async function prevFrame() {
  if (playing || currentFrame <= 1) return;

  const targetFrame = currentFrame - 2; // 要回退到的目标帧索引
  const keyFrameIndex = findPreviousKeyFrame(targetFrame + 1);

  if (keyFrameIndex === -1) return;

  await displayFramesInRange(keyFrameIndex, targetFrame + 1);
  currentFrame = targetFrame + 1; // 更新当前帧索引
}

5. 恢复播放时的全帧渲染 #

播放视频时，重置目标帧标记，让所有帧都正常渲染：

function playVideo() {
  if (playing) return;
  playing = true;
  targetRenderFrameIndex = null; // 播放时所有帧都渲染
  (async () => {
    for (let i = currentFrame; i < frames.length && playing; i++) {
      await videoDecoder.decode({...frames[i], _index: i});
      currentFrame = i + 1;
      await new Promise(r => setTimeout(r, frameDuration));
    }
    playing = false;
  })();
}

额外优化：关键帧缓存 #

如果视频关键帧间隔较小，可以提前解码关键帧并缓存为ImageBitmap，回退到关键帧时直接从缓存读取，减少重复解码开销：

const keyFrameCache = new Map(); // key: 关键帧索引, value: ImageBitmap

// 初始化时缓存所有关键帧
async function cacheKeyFrames() {
  const config = videoDecoder.state.configuration;
  for (let i = 0; i < frames.length; i++) {
    if (frames[i].type === 'key') {
      const frame = await new Promise(resolve => {
        const tempDecoder = new VideoDecoder({
          output: resolve,
          error: e => console.error(e)
        });
        tempDecoder.configure(config);
        tempDecoder.decode(frames[i]);
        tempDecoder.flush();
      });
      const bitmap = await createImageBitmap(frame);
      frame.close();
      keyFrameCache.set(i, bitmap);
    }
  }
}

之后在prevFrame中判断目标帧是否为关键帧，如果是直接从缓存渲染，跳过解码流程。

内容的提问来源于stack exchange，提问作者Daniel Robinson