JSMpeg播放WebSocket流无音频问题排查求助

我通过phoboslab/jsmpeg提供的Node.js WebSocket中继，结合FFmpeg将RTSP流推送到浏览器，使用JSMpeg播放时视频正常，但无音频输出。

使用的FFmpeg命令 #

ffmpeg -rtsp_transport tcp -i rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mp4 
       -f mpegts -c:v mpeg1video -c:a mp2 http://127.0.0.1:8081/stream_from_ffmpeg/<my_secret>

Node.js WebSocket中继代码 #

// Use the websocket-relay to serve a raw MPEG-TS over WebSockets. You can use
// ffmpeg to feed the relay. ffmpeg -> websocket-relay -> browser
// Example:
// node websocket-relay yoursecret 8081 8082
// ffmpeg -i <some input> -f mpegts http://localhost:8081/yoursecret

var fs = require('fs'),
    http = require('http'),
    WebSocket = require('ws');

if (process.argv.length < 3) {
    console.log(
        'Usage: \n' +
        'node websocket-relay.js <secret> [<stream-port> <websocket-port>]'
    );
    process.exit();
}

var STREAM_SECRET = process.argv[2],
    STREAM_PORT = process.argv[3] || 8081,
    WEBSOCKET_PORT = process.argv[4] || 8082,
    RECORD_STREAM = false;

// Websocket Server
var socketServer = new WebSocket.Server({port: WEBSOCKET_PORT, perMessageDeflate: false});
socketServer.connectionCount = 0;
socketServer.on('connection', function(socket, upgradeReq) {
    socketServer.connectionCount++;
    console.log(
        'New WebSocket Connection: ',
        (upgradeReq || socket.upgradeReq).socket.remoteAddress,
        (upgradeReq || socket.upgradeReq).headers['user-agent'],
        ('+socketServer.connectionCount+' total)'
    );
    socket.on('close', function(code, message){
        socketServer.connectionCount--;
        console.log(
            'Disconnected WebSocket ('+socketServer.connectionCount+' total)'
        );
    });
});
socketServer.broadcast = function(data) {
    socketServer.clients.forEach(function each(client) {
        if (client.readyState === WebSocket.OPEN) {
            client.send(data);
        }
    });
};

// HTTP Server to accept incoming MPEG-TS Stream from ffmpeg
var streamServer = http.createServer( function(request, response) {
    var params = request.url.substr(1).split('/');

    if (params[0] !== STREAM_SECRET) {
        console.log(
            'Failed Stream Connection: '+ request.socket.remoteAddress + ':' +
            request.socket.remotePort + ' - wrong secret.'
        );
        response.end();
    }

    response.connection.setTimeout(0);
    console.log(
        'Stream Connected: ' +
        request.socket.remoteAddress + ':' +
        request.socket.remotePort
    );
    request.on('data', function(data){
        socketServer.broadcast(data);
        if (request.socket.recording) {
            request.socket.recording.write(data);
        }
    });
    request.on('end',function(){
        console.log('close');
        if (request.socket.recording) {
            request.socket.recording.close();
        }
    });

    // Record the stream to a local file?
    if (RECORD_STREAM) {
        var path = 'recordings/' + Date.now() + '.ts';
        request.socket.recording = fs.createWriteStream(path);
    }
})
// Keep the socket open for streaming
streamServer.headersTimeout = 0;
streamServer.listen(STREAM_PORT);

console.log('Listening for incoming MPEG-TS Stream on http://127.0.0.1:'+STREAM_PORT+'/<secret>');
console.log('Awaiting WebSocket connections on ws://127.0.0.1:'+WEBSOCKET_PORT+'/');

前端播放代码 #

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <script src="./jsmpeg.min.js"></script>
    <title>Document</title>
  </head>
  <body>
    <canvas id="video-canvas"></canvas>
  </body>
  <script>
    let url;
    let player;
    let canvas = document.getElementById("video-canvas");
    let ipAddr = "127.0.0.1:8082";
    window.onload = async() => {
      url = `ws://${ipAddr}`;
      player = new JSMpeg.Player(url, { canvas: canvas, });
    };

  </script>
</html>

已尝试的修复操作 #

我尝试恢复JSMpeg的AudioContext状态，但问题未解决：

player.audioOut.context.onstatechange = async () => {
    console.log("Event triggered by audio");

    if (player.audioOut.context === "suspended") {
        await player.audioOut.context.resume();
    }
}

排查步骤及解决方案 #

• 规范FFmpeg音频编码参数
  JSMpeg仅支持MP2（MPEG-1 Layer II）音频，需明确指定兼容的采样率、比特率等参数，避免FFmpeg自动选择不兼容配置：

  ffmpeg -rtsp_transport tcp -i rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mp4 
         -f mpegts -c:v mpeg1video -b:v 1000k -r 25 -g 50 
         -c:a mp2 -b:a 128k -ar 44100 -ac 2 http://127.0.0.1:8081/stream_from_ffmpeg/<my_secret>
  

• 修复AudioContext恢复逻辑
  原代码判断状态的方式错误，且现代浏览器要求用户交互才能激活AudioContext，修改前端代码：

  window.onload = async() => {
    url = `ws://${ipAddr}`;
    player = new JSMpeg.Player(url, { 
      canvas: canvas,
      audio: true,
      autoplay: true
    });
  
    if (player.audioOut && player.audioOut.context) {
      // 监听状态变化恢复音频
      player.audioOut.context.onstatechange = () => {
        if (player.audioOut.context.state === 'suspended') {
          player.audioOut.context.resume();
        }
      };
  
      // 绑定页面点击事件触发音频激活
      document.body.addEventListener('click', () => {
        if (player.audioOut.context.state === 'suspended') {
          player.audioOut.context.resume();
        }
      });
    }
  };
  

• 验证流数据完整性

  1. 将FFmpeg输出的TS流保存到本地，用VLC播放确认是否有音频：

     ffmpeg -rtsp_transport tcp -i rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mp4 
            -f mpegts -c:v mpeg1video -c:a mp2 test.ts
     

  2. 启用中继的RECORD_STREAM = true，录制WebSocket转发的流，用VLC播放验证音频是否存在。

内容的提问来源于stack exchange，提问作者Nik