浏览器中合并Canvas与Webcam流生成视频的问题排查

需要在浏览器中生成由Canvas流（幻灯片）和Webcam流（视频）交替组成的视频，最终输出顺序为「幻灯片1+视频1+幻灯片2+视频2+幻灯片3+视频3」，但现有代码仅能合并前两个片段，后续片段无法正常播放，尝试过浏览器端FFmpeg也未解决。

const canvas = document.getElementById('myCanvas');
const ctx = canvas.getContext('2d');
const webcam = document.getElementById('webcam');
const videoPlayer = document.createElement('video');
videoPlayer.controls = true;
document.body.appendChild(videoPlayer);
const videoWidth = 640;
const videoHeight = 480;
let keepAnimating = true;
const frameRate=30;

// 获取摄像头权限
function setupWebcam() {
 const constraints = {
        video: {
             frameRate: frameRate,
            width: videoWidth,  
            height: videoHeight 
        }
    };
  navigator.mediaDevices.getUserMedia(constraints)
    .then(stream => {
      webcam.srcObject = stream;
      webcam.addEventListener('loadedmetadata', () => {
        recordSegments();
        console.log('摄像头已加载');
      });
    })
    .catch(err => {
      console.error("访问摄像头失败:", err);
      alert('无法访问摄像头，请检查权限设置后重试。');
    });
}

// Canvas动画绘制
function animateCanvas(content) {
  if (!keepAnimating) {
    console.log("keepAnimating", keepAnimating);
    return;
  };

  ctx.clearRect(0, 0, canvas.width, canvas.height);
  ctx.fillStyle = `rgba(${Math.floor(Math.random() * 255)}, ${Math.floor(Math.random() * 255)}, ${Math.floor(Math.random() * 255)}, 0.5)`;
  ctx.fillRect(0, 0, canvas.width, canvas.height);
  ctx.fillStyle = '#000';
  ctx.font = '48px serif';
  ctx.fillText(content + ' ' + new Date().toLocaleTimeString(), 50, 100);

  requestAnimationFrame(() => animateCanvas(content));
}

// 录制片段数组
const recordedSegments = [];

// 开始录制
function startRecording(stream, duration = 5000, content) {
  const recorder = new MediaRecorder(stream, { mimeType: 'video/webm' });
  const data = [];

  recorder.ondataavailable = e => data.push(e.data);

  keepAnimating = true;
  if (content) animateCanvas(content);
  recorder.start();
  
  return new Promise((resolve) => {
    setTimeout(() => {
      recorder.stop();
      keepAnimating = false;
    }, duration);

    recorder.onstop = () => {
      const blob = new Blob(data, { type: 'video/webm' });
      recordedSegments.push(blob);
      resolve(blob);
    };
  });
}

// 按顺序录制片段
async function recordSegments() {
  await startRecording(canvas.captureStream(frameRate), 2000, 'Canvas Draw 1').then(() => console.log('Canvas片段1录制完成'));
  await startRecording(webcam.srcObject, 3000).then(() => console.log('Webcam片段1录制完成'));
  await startRecording(canvas.captureStream(frameRate), 2000, 'Canvas Draw 2').then(() => console.log('Canvas片段2录制完成'));
  await startRecording(webcam.srcObject, 3000).then(() => console.log('Webcam片段2录制完成'));
  await startRecording(canvas.captureStream(frameRate), 2000, 'Canvas Draw 3').then(() => console.log('Canvas片段3录制完成'));
  await startRecording(webcam.srcObject, 3000).then(() => console.log('Webcam片段3录制完成'));
  
  mergeAndDownloadVideo();
}

// 下载视频
function downLoadVideo(blob){
 const url = URL.createObjectURL(blob);
  const a = document.createElement('a');
  a.style.display = 'none';
  a.href = url;
  a.download = 'merged-video.webm';
  document.body.appendChild(a);
  a.click();

  setTimeout(() => {
    document.body.removeChild(a);
    window.URL.revokeObjectURL(url);
  }, 100);
}

// 合并并下载视频（原方法存在问题）
function mergeAndDownloadVideo() {
  console.log("已录制片段数量", recordedSegments.length);
  const superBlob = new Blob(recordedSegments, { type: 'video/webm' });
  downLoadVideo(superBlob)
}

// 启动流程
setupWebcam();

直接将多个独立的WebM格式Blob拼接成一个Blob是无效的——每个WebM文件都包含完整的文件头和元数据，播放器只能识别第一个文件的结构，后续片段的二进制内容无法被解析。

MediaSource API允许将多个媒体片段依次添加到同一个媒体源中，确保播放器能正确解析整个视频流。修改后的合并逻辑如下：

async function mergeAndDownloadVideo() {
  console.log("已录制片段数量", recordedSegments.length);
  
  const mediaSource = new MediaSource();
  const videoUrl = URL.createObjectURL(mediaSource);
  
  return new Promise((resolve) => {
    mediaSource.addEventListener('sourceopen', async () => {
      // 确保编码格式与录制时一致
      const sourceBuffer = mediaSource.addSourceBuffer('video/webm; codecs="vp8, vorbis"');
      
      // 依次添加每个片段
      for (const blob of recordedSegments) {
        await new Promise((resolveSegment) => {
          sourceBuffer.addEventListener('updateend', resolveSegment, { once: true });
          sourceBuffer.appendBuffer(await blob.arrayBuffer());
        });
      }
      
      // 标记媒体源结束
      mediaSource.endOfStream();
      
      // 将合并后的媒体流转为Blob下载
      const recorder = new MediaRecorder(mediaSource);
      const chunks = [];
      recorder.ondataavailable = (e) => chunks.push(e.data);
      recorder.onstop = () => {
        const mergedBlob = new Blob(chunks, { type: 'video/webm' });
        downLoadVideo(mergedBlob);
        URL.revokeObjectURL(videoUrl);
        resolve();
      };
      
      recorder.start();
      setTimeout(() => recorder.stop(), 100); // 确保录制完成
    });
  });
}

如果需要更稳定的跨格式支持，可使用FFmpeg.wasm处理合并，核心逻辑如下：

// 需先引入FFmpeg.wasm库
async function mergeWithFFmpeg() {
  const ffmpeg = createFFmpeg({ log: true });
  await ffmpeg.load();
  
  // 将每个片段写入虚拟文件系统
  for (let i = 0; i < recordedSegments.length; i++) {
    const file = new File([recordedSegments[i]], `segment${i}.webm`);
    ffmpeg.FS('writeFile', `segment${i}.webm`, await file.arrayBuffer());
  }
  
  // 生成concat文件列表
  const concatContent = recordedSegments.map((_, i) => `file 'segment${i}.webm'`).join('\n');
  ffmpeg.FS('writeFile', 'concat.txt', concatContent);
  
  // 执行合并命令（直接复制编码，无重新编码损耗）
  await ffmpeg.run('-f', 'concat', '-safe', '0', '-i', 'concat.txt', '-c', 'copy', 'merged.webm');
  
  // 读取合并后的文件并下载
  const data = ffmpeg.FS('readFile', 'merged.webm');
  const mergedBlob = new Blob([data.buffer], { type: 'video/webm' });
  downLoadVideo(mergedBlob);
  
  // 清理虚拟文件系统
  recordedSegments.forEach((_, i) => ffmpeg.FS('unlink', `segment${i}.webm`));
  ffmpeg.FS('unlink', 'concat.txt');
  ffmpeg.FS('unlink', 'merged.webm');
}

内容的提问来源于stack exchange，提问作者Stefdelec