iOS Safari下MediaStream加音频轨道致MediaRecorder无数据问题解决

基于Angular 15 & Ionic 6的网站，支持用户上传视频后在Canvas上编辑，通过捕获Canvas流并混合原视频音频轨道，最终用MediaRecorder导出处理后的视频文件。该流程在Windows桌面Chrome中正常运行，但在iOS Safari（测试版本16.2）中，若为MediaStream添加音频轨道，MediaRecorder录制结果为0字节。

现有代码片段 #

HTML元素 #

<video id="uploadedVideo" #uploadedVideo autoplay webkit-playsinline playsinline style="display: none;"></video>
<canvas id="canvasVideo" #canvasVideo style="display: none;"></canvas>

元素获取代码 #

@ViewChild('uploadedVideo', { static: false }) videoElementRef: ElementRef<HTMLVideoElement>;
video: HTMLVideoElement;
@ViewChild('fileUpload', { static: false }) fileUpload: ElementRef;
@ViewChild('canvasVideo', { static: false }) videoCanvasElementRef: ElementRef<HTMLCanvasElement>;
videoCanvas: HTMLCanvasElement;
videoCanvasCtx: CanvasRenderingContext2D;

视频静音初始化 #

silenceVideoAudio() {
    if (this.videoConnectedToAudioContext) {
      return;
    }
    this.audioContext = new AudioContext();
    this.sourceNode = this.audioContext.createMediaElementSource(this.video);
    const gainNode = this.audioContext.createGain();
    gainNode.gain.value = 0; // 确保视频音频静音
    this.sourceNode.connect(gainNode);
    gainNode.connect(this.audioContext.destination);
    this.videoConnectedToAudioContext = true;
}

文件选择处理 #

onVideoFileSelected(file: File) {
  this.video.src = URL.createObjectURL(file);
  this.silenceVideoAudio();
  this.video.onloadeddata = async () => {
    await this.startVideoProcessing();
  };
  this.video.onended = async () => {
    this.recorder.stop();
  };
}

核心录制处理 #

async startVideoProcessing() {
    // 其他无关初始化逻辑

    let stream: MediaStream;
    if (this.isSafari) {
      stream = new MediaStream([...this.videoCanvas.captureStream(30).getVideoTracks()]);
    } else {
      stream = new MediaStream([...this.videoCanvas.captureStream(30).getVideoTracks(), this.getAudioTrack()]);
    }
    const options = {
        audioBitsPerSecond: 128000,
        videoBitsPerSecond: 2500000,
        mimeType: this.mimeType,
    };
    this.recorder = new MediaRecorder(stream, options);
    this.recorder.ondataavailable = (event) => {
        if (event.data && event.data.size > 0) {
            this.chunks.push(event.data);
        }
    };
    this.recorder.start();
    this.recorder.onerror = (event) => {
      console.error(event);
      throw new Error(event.toString());
    };
    this.recorder.onstop = () => {
      const videoBlob = new Blob(this.chunks, { type: this.mimeType });
      const videoFile = new File([videoBlob], this.mimeType === 'video/mp4' ? 'video-green-screen.mp4' : 'video-green-screen.webm', { type: this.mimeType });
      this.fileSelected.emit(videoFile);
    };
  }

音频轨道获取方法 #

getAudioTrack() {
    if ((this.video as any).captureStream) {
      this.videoAudioTracks = (this.video as any).captureStream().getAudioTracks()[0];
    } else if ((this.video as any).mozCaptureStream) {
      this.videoAudioTracks = (this.video as any).mozCaptureStream().getAudioTracks()[0];
    } else {
      const destination = this.audioContext.createMediaStreamDestination();
      this.sourceNode.connect(destination);
      this.videoAudioTracks = destination.stream.getAudioTracks()[0];
    }
    return this.videoAudioTracks;
  }

需求 #

1. 为Canvas捕获的流添加原视频音频轨道
2. 兼容iOS Safari（尽可能支持16.x之前版本）、Chrome、Firefox

iOS Safari录制失败原因分析 #

1. 音视频轨道时间同步要求严格：iOS Safari的MediaRecorder要求音视频轨道的时间戳完全对齐，直接从video元素捕获的音频轨道与Canvas生成的视频轨道可能存在时间偏移，导致录制逻辑中断，最终输出空数据。
2. AudioContext状态限制：iOS Safari要求AudioContext必须通过用户交互激活，若上下文处于挂起状态，通过其创建的音频轨道无法被MediaRecorder正确捕获。
3. MediaStream轨道添加时机问题：iOS Safari中，MediaRecorder初始化后添加轨道可能不被支持，且要求轨道在流创建时已处于活跃状态，否则录制会失败。

解决方案 #

1. 优化音频轨道获取逻辑 #

确保AudioContext已激活，同时处理不同浏览器的轨道捕获方式：

async getAudioTrack(): Promise<MediaStreamTrack> {
  // 激活挂起的AudioContext（iOS要求用户交互触发）
  if (this.audioContext && this.audioContext.state === 'suspended') {
    await this.audioContext.resume();
  }

  // 优先使用video原生captureStream
  if ('captureStream' in this.video) {
    const videoStream = (this.video as any).captureStream();
    const audioTracks = videoStream.getAudioTracks();
    if (audioTracks.length > 0) {
      return audioTracks[0];
    }
  } else if ('mozCaptureStream' in this.video) {
    const videoStream = (this.video as any).mozCaptureStream();
    const audioTracks = videoStream.getAudioTracks();
    if (audioTracks.length > 0) {
      return audioTracks[0];
    }
  }

  // 回退到AudioContext方式，同时保持静音设置
  const destination = this.audioContext.createMediaStreamDestination();
  this.sourceNode.disconnect();
  this.sourceNode.connect(destination);
  
  // 重新连接静音节点
  const gainNode = this.audioContext.createGain();
  gainNode.gain.value = 0;
  this.sourceNode.connect(gainNode);
  gainNode.connect(this.audioContext.destination);

  return destination.stream.getAudioTracks()[0];
}

2. 调整流与MediaRecorder初始化流程 #

确保音视频轨道同时添加到流中，并等待轨道就绪后启动录制：

async startVideoProcessing() {
  // 等待视频进入可播放状态，确保轨道活跃
  await new Promise(resolve => {
    if (this.video.readyState >= 2) resolve(null);
    else this.video.oncanplay = resolve;
  });

  const videoTracks = this.videoCanvas.captureStream(30).getVideoTracks();
  let audioTrack: MediaStreamTrack | null = null;

  try {
    audioTrack = await this.getAudioTrack();
  } catch (error) {
    console.warn('获取音频轨道失败:', error);
  }

  // 组合音视频轨道
  const streamTracks = [...videoTracks];
  if (audioTrack) {
    streamTracks.push(audioTrack);
  }

  const stream = new MediaStream(streamTracks);

  // iOS Safari强制使用video/mp4格式
  const options = {
    audioBitsPerSecond: 128000,
    videoBitsPerSecond: 2500000,
    mimeType: this.isSafari ? 'video/mp4' : this.mimeType,
  };

  this.recorder = new MediaRecorder(stream, options);
  this.chunks = [];

  this.recorder.ondataavailable = (event) => {
    if (event.data && event.data.size > 0) {
      this.chunks.push(event.data);
    }
  };

  this.recorder.onerror = (event) => {
    console.error('MediaRecorder错误:', event);
    stream.getTracks().forEach(track => track.stop());
  };

  this.recorder.onstop = () => {
    stream.getTracks().forEach(track => track.stop());
    
    // 处理空数据情况，降级为仅视频录制
    if (this.chunks.length === 0) {
      console.error('无录制数据，降级为仅视频输出');
      const videoOnlyStream = new MediaStream(videoTracks);
      const fallbackRecorder = new MediaRecorder(videoOnlyStream, {
        videoBitsPerSecond: 2500000,
        mimeType: this.isSafari ? 'video/mp4' : this.mimeType,
      });

      fallbackRecorder.ondataavailable = (e) => {
        if (e.data.size > 0) {
          const videoBlob = new Blob([e.data], { type: options.mimeType });
          const videoFile = new File([videoBlob], 'video-green-screen.mp4', { type: options.mimeType });
          this.fileSelected.emit(videoFile);
        }
      };

      fallbackRecorder.start();
      setTimeout(() => fallbackRecorder.stop(), 100);
      return;
    }

    const videoBlob = new Blob(this.chunks, { type: options.mimeType });
    const videoFile = new File([videoBlob], options.mimeType === 'video/mp4' ? 'video-green-screen.mp4' : 'video-green-screen.webm', { type: options.mimeType });
    this.fileSelected.emit(videoFile);
  };

  // 延迟启动，确保轨道状态稳定
  await new Promise(resolve => setTimeout(resolve, 100));
  this.recorder.start();
}

3. 修复AudioContext激活时机 #

在用户交互（文件选择）时确保AudioContext已激活：

async silenceVideoAudio() {
  if (this.videoConnectedToAudioContext) {
    return;
  }
  this.audioContext = new AudioContext();
  // 激活上下文（iOS要求用户交互触发）
  if (this.audioContext.state === 'suspended') {
    await this.audioContext.resume();
  }
  this.sourceNode = this.audioContext.createMediaElementSource(this.video);
  const gainNode = this.audioContext.createGain();
  gainNode.gain.value = 0;
  this.sourceNode.connect(gainNode);
  gainNode.connect(this.audioContext.destination);
  this.videoConnectedToAudioContext = true;
}

4. 额外兼容性优化 #

• 保持video元素的webkit-playsinline和playsinline属性，确保iOS上视频可正常播放
• 录制完成后主动停止所有轨道，释放系统资源
• 添加空数据降级逻辑，避免无输出情况

内容的提问来源于stack exchange，提问作者Eric Belisle Giddings