Node.js中Twilio双向流问题：ElevenLabs音频流无法播放

需求 #

• 将静态文本通过WebSocket发送给ElevenLabs进行TTS，接收其返回的ulaw_8000格式音频流
• 将该音频流推送给Twilio，作为来电者的语音回复播放

现有测试代码 #

import { createServer } from 'http';
import express from 'express';
import { WebSocketServer, WebSocket } from 'ws';
import 'dotenv/config';
import voiced from 'twilio'

const VoiceResponse = voiced.twiml.VoiceResponse;

const voiceId = "somevoiceid";
const model = 'eleven_monolingual_v1';

const ngrokURL = 'https://myngrok-url-here.app'

let streamsID = '';


const app = express();
app.use(express.urlencoded({ extended: true }));

const server = createServer(app);


app.get('/', (_, res) => res.type('text').send('Twilio media stream transcriber'));

app.post('/statusCallBack', (req, res) => {
    console.log(req.body)
    streamsID = req.body.StreamSid;
    console.log(streamsID);
});


// 处理来电并收集语音输入
app.post('/', (req, res) => {
    const response = new VoiceResponse();
    const connect = response.connect();
    connect.stream({
        url: `${ngrokURL}`,
        statusCallback: `${ngrokURL}/statusCallBack`,
        statusCallbackMethod: "POST"
    });

    console.log(response.toString());

    const gather = response.gather({
        input: 'speech',
        timeout: 3,
        action: '/process_speech',
        method: 'POST',
    });
    gather.say('Thank you for calling. How may you be assisted?');

    response.say("I didn't catch that. Please try again.");

    res.type('xml').send(response.toString());
});


// 处理语音输入并回复
app.post('/process_speech', (req, res) => {
    const speechResult = req.body.SpeechResult;
    const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${model}&output_format=ulaw_8000`;
    const socket = new WebSocket(wsUrl);
    const response = new VoiceResponse();

    if (speechResult) {
        response.say("Here is a poem to soothe you.")
        socket.onopen = function (event) {
            const bosMessage = {
                "text": " ",
                "voice_settings": {
                    "stability": 0.5,
                    "similarity_boost": 0.8
                },
                "xi_api_key": process.env.ELEVENLABS_API_KEY,
            };

            socket.send(JSON.stringify(bosMessage));

            console.log(`SpeechResult: ${req.body.SpeechResult}`);

            const textMessage = {
                "text": "roses are red,\n" +
                    "violets are blue\n" +
                    "You're an idiot but I still love you\n"
            };

            socket.send(JSON.stringify(textMessage));

            const eosMessage = {
                "text": ""
            };

            socket.send(JSON.stringify(eosMessage));
        };

        socket.onmessage = function (event) {
            const responded = JSON.parse(event.data);

            if (responded.audio) {
                console.log("Received audio chunk");

                const ulaw_data_buffer = Buffer.from(responded.audio);
                const payload = ulaw_data_buffer.toString('base64');

                console.log("Trying to send...")
                let load = {
                    "event": "media",
                    "streamSid": streamsID,
                    "media": {
                        "payload": payload
                    }
                }

                console.log(load)

                socketTwilio.send(JSON.stringify(load));

            } else {
                console.log("No audio data in the response");
            }

            if (responded.isFinal) {
                // 生成完成
            }

            if (responded.normalizedAlignment) {
                // 使用对齐信息
            }
        };

        socket.onerror = function (error) {
            console.error(`WebSocket Error: ${error}`);
        };

        socket.onclose = function (event) {
            if (event.wasClean) {
                console.info(`Connection closed cleanly, code=${event.code}, reason=${event.reason}`);
                const gather = response.gather({
                    input: 'speech',
                    timeout: 3,
                    action: '/follow_up',
                    method: 'POST',
                });
                gather.say('Will there be anything else?');
                res.type('xml').send(response.toString());
            } else {
                console.warn('Connection died');
            }
        };

    } else {
        response.say("Sorry, I didn't get that. Please try speaking again.");
    }
});

app.post('/follow_up', (req, res) => {
    const speechResult = req.body.SpeechResult.toLowerCase();
    const response = new VoiceResponse();

    if (speechResult.includes('yes')) {
        response.redirect({ method: 'POST' }, '/');
    } else {
        response.say('Thank you for calling. Goodbye!');
    }

    res.type('xml').send(response.toString());
});

socketTwilio.addEventListener('open', function (event) {
    console.log("Twilio Websocket opened");
});

const wss = new WebSocketServer({ server });
wss.on('connection', (ws) => {
    console.log('Twilio media stream WebSocket connected');
    ws.on('message', (message) => {
        console.log("Twilio: ", message)
    });
    ws.on('close', () => {
        console.log('Twilio media stream WebSocket disconnected');
    });
});

console.log('Listening on port 8080');
server.listen(8080);

当前问题 #

程序无报错，通话流程正常，Twilio可执行指定语音播报，但ElevenLabs返回的音频流无法播放，仅短暂静默后直接执行后续gather.say('Will there be anything else?')。已确认：

• ElevenLabs正常返回音频流
• Twilio媒体流WebSocket已成功连接并能接收回调

核心问题分析及解决方案 #

1. 未定义的socketTwilio变量 #

代码中直接使用socketTwilio.send()但未对该变量赋值，Twilio的媒体流WebSocket连接是通过wss.on('connection')建立的，需将连接实例保存到全局变量：

// 全局定义变量保存Twilio连接
let twilioMediaSocket = null;

// 修改WebSocket服务器逻辑
const wss = new WebSocketServer({ server });
wss.on('connection', (ws) => {
    console.log('Twilio media stream WebSocket connected');
    twilioMediaSocket = ws; // 保存连接实例
    ws.on('message', (message) => {
        console.log("Twilio: ", message.toString())
    });
    ws.on('close', () => {
        console.log('Twilio media stream WebSocket disconnected');
        twilioMediaSocket = null; // 连接关闭后清空
    });
});

// 在ElevenLabs的onmessage中使用该变量发送音频
socket.onmessage = function (event) {
    const responded = JSON.parse(event.data);
    if (responded.audio && twilioMediaSocket && twilioMediaSocket.readyState === WebSocket.OPEN) {
        const load = {
            "event": "media",
            "streamSid": streamsID,
            "media": {
                "payload": responded.audio // 直接使用ElevenLabs返回的base64
            }
        };
        twilioMediaSocket.send(JSON.stringify(load));
    }
};

2. 音频数据重复编码 #

ElevenLabs返回的responded.audio已经是base64编码的ulaw_8000格式数据，当前代码中Buffer.from(responded.audio)再转base64属于重复编码，导致格式错误，Twilio无法解析。直接使用原始返回值即可。

3. TwiML逻辑冲突 #

初始通话同时使用<Connect><Stream>和<Gather>会导致逻辑冲突：<Gather>会接管通话等待用户输入，而<Stream>处于闲置状态。调整流程：

• 初始接口仅保留<Gather>，去掉<Connect><Stream>
• 在/process_speech中返回包含<Connect><Stream>的TwiML，让Twilio建立媒体流连接后再推送音频
• 音频推送完成后，通过Twilio Calls API更新通话TwiML，触发后续询问

修改后的初始接口：

app.post('/', (req, res) => {
    const response = new VoiceResponse();
    const gather = response.gather({
        input: 'speech',
        timeout: 3,
        action: '/process_speech',
        method: 'POST',
    });
    gather.say('Thank you for calling. How may you be assisted?');
    response.say("I didn't catch that. Please try again.");
    res.type('xml').send(response.toString());
});

修改后的/process_speech接口（需配置Twilio账号信息）：

app.post('/process_speech', (req, res) => {
    const speechResult = req.body.SpeechResult;
    const response = new VoiceResponse();

    if (speechResult) {
        // 返回Stream TwiML，让Twilio建立媒体流连接
        const connect = response.connect();
        connect.stream({
            url: `${ngrokURL}`,
            statusCallback: `${ngrokURL}/statusCallBack`,
            statusCallbackMethod: "POST"
        });
        res.type('xml').send(response.toString());

        // 延迟启动ElevenLabs请求，确保Twilio连接建立完成
        setTimeout(() => {
            const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${model}&output_format=ulaw_8000`;
            const socket = new WebSocket(wsUrl);

            socket.onopen = function (event) {
                const bosMessage = {
                    "text": " ",
                    "voice_settings": {
                        "stability": 0.5,
                        "similarity_boost": 0.8
                    },
                    "xi_api_key": process.env.ELEVENLABS_API_KEY,
                };
                socket.send(JSON.stringify(bosMessage));

                const textMessage = {
                    "text": "roses are red,\nviolets are blue\nYou're an idiot but I still love you\n"
                };
                socket.send(JSON.stringify(textMessage));

                const eosMessage = { "text": "" };
                socket.send(JSON.stringify(eosMessage));
            };

            socket.onmessage = function (event) {
                const responded = JSON.parse(event.data);
                if (responded.audio && twilioMediaSocket && twilioMediaSocket.readyState === WebSocket.OPEN) {
                    const load = {
                        "event": "media",
                        "streamSid": streamsID,
                        "media": {
                            "payload": responded.audio
                        }
                    };
                    twilioMediaSocket.send(JSON.stringify(load));
                }
            };

            socket.onclose = function (event) {
                if (event.wasClean) {
                    // 音频推送完成后，更新通话TwiML触发后续询问
                    fetch(`https://api.twilio.com/2010-04-01/Accounts/${process.env.TWILIO_ACCOUNT_SID}/Calls/${req.body.CallSid}.json`, {
                        method: 'POST',
                        headers: {
                            'Content-Type': 'application/x-www-form-urlencoded',
                            'Authorization': 'Basic ' + Buffer.from(`${process.env.TWILIO_ACCOUNT_SID}:${process.env.TWILIO_AUTH_TOKEN}`).toString('base64')
                        },
                        body: new URLSearchParams({
                            'Url': `${ngrokURL}/follow_up_prompt`
                        })
                    });
                }
            };

            socket.onerror = function (error) {
                console.error(`ElevenLabs WebSocket Error: ${error}`);
            };
        }, 1000);
    } else {
        response.say("Sorry, I didn't get that. Please try speaking again.");
        res.type('xml').send(response.toString());
    }
});

// 新增推送完成后的提示接口
app.post('/follow_up_prompt', (req, res) => {
    const response = new VoiceResponse();
    const gather = response.gather({
        input: 'speech',
        timeout: 3,
        action: '/follow_up',
        method: 'POST',
    });
    gather.say('Will there be anything else?');
    res.type('xml').send(response.toString());
});

内容的提问来源于stack exchange，提问作者itwabi