Twilio语音通话中无法听到WebSocket发送的音频消息

我正在开发一个集成Twilio Voice和Google Cloud Speech-to-Text的Node.js WebSocket服务器，目标是通过WebSocket回流传音频消息到Twilio语音通话中，但发送后无法在通话中听到音频。

require("dotenv").config();
const WebSocket = require("ws");
const twilio = require("twilio");
const speech = require("@google-cloud/speech");
const fs = require("fs");
const ffmpeg = require("fluent-ffmpeg");
const clientGoogle = new speech.SpeechClient();

process.env.GOOGLE_APPLICATION_CREDENTIALS = `${__dirname}/seismic-iridium-434511-n8-cc9b62d1aeac.json`;

const accountSid = process.env.ACCOUNTSID;
const authToken = process.env.AUTHTOKEN;
const client = twilio(accountSid, authToken);

const wss = new WebSocket.Server({ port: 8080 });

const request = {
  config: {
    encoding: "MULAW",
    sampleRateHertz: 8000,
    languageCode: "en-GB",
  },
  interimResults: true,
};

wss.on("connection", (ws) => {
  console.log("WebSocket connection established.");

  let recognizeStream;
  let silenceTimeout;

  ws.on("message", (message) => {
    const msg = JSON.parse(message);

    switch (msg.event) {
      case "connected":
        console.log(`A new call has connected.`);
        break;
      case "start":
        console.log(`Starting Media Stream ${msg.streamSid}`);

        sendAudioResponse(ws, msg.streamSid);

        recognizeStream = clientGoogle
          .streamingRecognize(request)
          .on("error", (err) => {
            console.error("Error in recognition stream:", err);
            recognizeStream = null;
          })
          .on("data", (data) => {
            const transcript = data.results[0].alternatives[0].transcript;
            console.log(transcript);
          });
        break;
      case "media":
        if (recognizeStream) {
          recognizeStream.write(msg.media.payload);
        } else {
          console.warn("Media received but no active recognize stream.");
        }
        break;
      case "stop":
        console.log(`Call Has Ended`);
        if (recognizeStream) {
          recognizeStream.destroy();
          recognizeStream = null;
        }
        clearTimeout(silenceTimeout);
        break;
    }
  });

  ws.on("close", () => {
    console.log("WebSocket connection closed.");
    if (recognizeStream) {
      recognizeStream.destroy();
      recognizeStream = null;
    }
    clearTimeout(silenceTimeout);
  });
});

function convertWavToMulaw(wavFilePath, outputFilePath, callback) {
  ffmpeg(wavFilePath)
    .audioCodec("pcm_mulaw")
    .audioFrequency(8000)
    .format("mulaw"); // Explicitly specify the format
    .on("end", function () {
      console.log("Conversion to mulaw completed.");
      callback(null, outputFilePath);
    })
    .on("error", function (err) {
      console.error("Error during conversion:", err);
      callback(err);
    })
    .save(outputFilePath);
}

function removeHeadersAndEncode(mulawFilePath) {
  const audioBuffer = fs.readFileSync(mulawFilePath);
  return audioBuffer.toString("base64"); // Encode the raw audio data to base64
}

function sendAudioResponse(ws, streamSid) {
  const wavFilePath = `${__dirname}/a.wav`;
  const rawFilePath = `${__dirname}/a.raw`; // Use .raw for the output file

  convertWavToMulaw(wavFilePath, rawFilePath, (err) => {
    if (err) {
      console.error("Failed to convert WAV to mulaw.");
      return;
    }

    const base64Payload = removeHeadersAndEncode(rawFilePath);

    ws.send(
      JSON.stringify({
        event: "media",
        streamSid: streamSid,
        media: {
          payload: base64Payload,
        },
      })
    );
  });
}

console.log(
  "WebSocket server is listening on ws://rnhcn-2405-201-682f-e812-e9db-c56d-9ca1-f001.a.free.pinggy.link"
);

async function makeCallWithMediaStream(toNumber, fromNumber) {
  try {
    const call = await client.calls.create({
      to: toNumber,
      from: fromNumber,
      twiml: `<Response>
                <Start>
                  <Stream url="wss://rnhcn-2405-201-682f-e812-e9db-c56d-9ca1-f001.a.free.pinggy.link"/>
                </Start>
                <Say>Hello! This call is being monitored for a demonstration.</Say>
                <Pause length="30" />
              </Response>`,
    });

    console.log(`Call initiated with SID: ${call.sid}`);
  } catch (error) {
    console.error(`Error making call: ${error.message}`);
  }
}

makeCallWithMediaStream("+918888888888", "+19876543210");

根据Twilio文档，media.payload应为原始MULAW/8000音频，经base64编码且不能包含任何音频文件头字节。我已尝试将.wav文件转换为MULAW格式、移除文件头并将原始音频数据编码为base64，但音频消息仍无法在通话中播放。我的实现中遗漏或错误了什么？恳请指导！

1. 修复FFmpeg转换代码的语法错误 #

convertWavToMulaw函数中，.format("mulaw");后的分号破坏了链式调用，导致后续的on事件监听无法绑定到FFmpeg进程，转换过程完全失效。修正后的代码：

function convertWavToMulaw(wavFilePath, outputFilePath, callback) {
  ffmpeg(wavFilePath)
    .audioCodec("pcm_mulaw")
    .audioFrequency(8000)
    .format("mulaw") // 移除此处的分号
    .on("end", function () {
      console.log("Conversion to mulaw completed.");
      callback(null, outputFilePath);
    })
    .on("error", function (err) {
      console.error("Error during conversion:", err);
      callback(err);
    })
    .save(outputFilePath);
}

2. 强制输出纯原始MULAW数据 #

部分FFmpeg版本使用.format("mulaw")仍会生成带文件头的文件，改用rawvideo格式可确保输出纯原始音频数据，同时强制单声道（Twilio语音通话要求单声道）：

function convertWavToMulaw(wavFilePath, outputFilePath, callback) {
  ffmpeg(wavFilePath)
    .audioCodec("pcm_mulaw")
    .audioFrequency(8000)
    .audioChannels(1)
    .format("rawvideo")
    .on("end", function () {
      console.log("Conversion to mulaw completed.");
      callback(null, outputFilePath);
    })
    .on("error", function (err) {
      console.error("Error during conversion:", err);
      callback(err);
    })
    .save(outputFilePath);
}

3. 调整音频发送时机 #

在start事件触发时立即发送音频，可能此时通话的媒体流尚未完全就绪，建议延迟几百毫秒再发送：

case "start":
  console.log(`Starting Media Stream ${msg.streamSid}`);
  // 延迟500ms确保媒体流建立完成
  setTimeout(() => {
    sendAudioResponse(ws, msg.streamSid);
  }, 500);
  // 剩余代码保持不变

4. 验证转换后的音频数据 #

将转换后的base64数据解码为.raw文件，用Audacity等工具导入（选择MULAW编码、8000Hz采样率、单声道），确认音频本身可正常播放，排除源文件或转换环节的问题。

内容的提问来源于stack exchange，提问作者hamid saifi