通过WebSocket向TwiML双向通话发送媒体消息无声音问题

我正在开发一个脚本，流程为：用Google Cloud Text-to-Speech生成LINEAR16编码的音频，通过WaveFile库转成mulaw/8000格式并转为base64，最后发送到WebSocket服务器，但通话中发送媒体消息时完全没有声音（连杂音都没有）。

我执行了以下步骤：

1. Node.js WebSocket服务器代码 #

const WebSocket = require("ws");
const wss = new WebSocket.Server({ port: 8080 });

const path = require("path");

wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");

    ws.on("message", function incoming(message) {
        let msg;
        try {
            msg = JSON.parse(message);
        } catch (e) {
            console.error('Error parsing message:', e);
            return;
        }
        switch (msg.event) {
            case "connected":
                console.log(`A new call has connected.`);
                break;
            case "start":
                console.log(`Starting Media Stream ${msg.streamSid}`);
                break;
            case "media":
                // Write Media Packets to the recognize stream
                if (msg.test == 1)
                    console.log(msg)
                break;
            case "stop":
                console.log(`Call Has Ended`);
                break;
        }
    });
});

console.log("Listening on Port 8080");

2. 用TwiML API发起通话（PHP代码） #

$sid = "SID_HERE";
$token = "TOKEN_HERE";

$client = new Twilio\Rest\Client($sid, $token);

$xml ='
<Response>
    <Say>This is your Voice Assistant speaking!</Say>
    <Connect>
        <Stream url="wss://MY_WEBSOCKET_SERVER_HERE">
            <Parameter name="aCutomParameter" value="aCustomValue that was set in TwiML" />
        </Stream>
    </Connect>
</Response>';


$call = $client->account->calls->create(
    '+TARGET_NUMBER_HERE',
    '+TWILIO_NUMBER_HERE',
    [
        'twiml' => $xml
    ]
);

3. 生成并发送TTS音频的Node.js代码 #

const textToSpeech = require('@google-cloud/text-to-speech');
const WebSocket = require('ws');
const ttsClient = new textToSpeech.TextToSpeechClient();
const wavefile = require('wavefile');
const fs = require('fs');

const path = require("path");

require("dotenv").config();

async function generateTtsAudio(text) {
    const request = {
        input: { text: text },
        voice: { languageCode: 'en-US', ssmlGender: 'NEUTRAL' },
        audioConfig: { audioEncoding: 'LINEAR16', sampleRateHertz: 8000 },
    };

    const [response] = await ttsClient.synthesizeSpeech(request);
    return response.audioContent;
}

async function sendTtsToWebSocket(text, wsUrl, streamSid) {
    const audioContent = await generateTtsAudio(text);

    const wav = new wavefile.WaveFile(audioContent);
        wav.toBitDepth('8');
        wav.toSampleRate(8000);
        av.toMuLaw();

    const base64Audio = Buffer.from(wav.data.samples).toString('base64');
    console.log(base64Audio);

    const ws = new WebSocket(wsUrl);

    ws.on('open', function open() {
        const message = {
            event: 'media',
            streamSid: streamSid,
            media: { payload: base64Audio }
        };

        const markMessage = {
            event: 'mark',
            streamSid: streamSid,
            "mark": {
                name: "testmark"
            }
        };

        ws.send(JSON.stringify(message));
        ws.send(JSON.stringify(markMessage));
    });

    ws.on('error', function error(error) {
        console.error('WebSocket Error:', error);
    });
}

const wsUrl = 'wss://MY_WEBSOCKET_SERVER_HERE';
const streamSid = 'STREAM_SID_HERE'; // 从通话启动时获取
const text = 'Hello, this is a TTS test message.';

sendTtsToWebSocket(text, wsUrl, streamSid)
    .catch(console.error);

问题排查与修复方案 #

1. 代码拼写错误 #

音频转换部分存在变量名拼写错误：av.toMuLaw()应为wav.toMuLaw()，这会导致mulaw转换步骤失败，直接跳过音频格式处理。

2. 音频数据处理错误 #

WaveFile的toMuLaw()转换后，音频数据是二进制流而非样本数组，无需从wav.data.samples提取，直接使用wav.toBuffer()获取转换后的二进制数据：

const wav = new wavefile.WaveFile(audioContent);
wav.toBitDepth('8');
wav.toSampleRate(8000);
wav.toMuLaw();

// 正确获取mulaw编码后的二进制数据并转base64
const base64Audio = wav.toBuffer().toString('base64');

3. WebSocket连接复用问题 #

Twilio Media Stream要求复用Twilio主动发起的WebSocket连接，而非新建连接。修改WebSocket服务器代码保存活跃连接，并通过该连接发送媒体消息：

// WebSocket服务器中添加活跃连接存储
let activeCallConnection = null;

wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");
    activeCallConnection = ws;
    
    ws.on("close", () => {
        activeCallConnection = null;
    });

    // ... 原有消息处理逻辑
});

// 导出连接实例供TTS模块使用
module.exports = { activeCallConnection };

修改TTS发送函数，复用已有连接：

const { activeCallConnection } = require('./你的WebSocket服务器文件名');

async function sendTtsToWebSocket(text, streamSid) {
    if (!activeCallConnection) {
        console.error("无活跃通话连接");
        return;
    }

    // ... 原有音频生成与转换逻辑

    const message = {
        event: 'media',
        streamSid: streamSid,
        media: { payload: base64Audio }
    };

    activeCallConnection.send(JSON.stringify(message));
}

4. 明确音频声道数 #

在Google TTS请求中指定单声道（Twilio要求单声道音频）：

const request = {
    input: { text: text },
    voice: { languageCode: 'en-US', ssmlGender: 'NEUTRAL' },
    audioConfig: { 
        audioEncoding: 'LINEAR16', 
        sampleRateHertz: 8000,
        audioChannelCount: 1 // 指定单声道
    },
};

内容的提问来源于stack exchange，提问作者Michael Naidis