Twilio <Stream>对接Azure连续语音转文字无转录结果问题咨询

错误原因 #

• 音频格式转换错误：mulaw-js解码后返回的是PCM采样值数组，Azure语音SDK要求传入小端序16位整数格式的Buffer，直接写入数组会导致音频无法被识别
• 实例复用逻辑错误：全局共用同一个推送流和识别器实例，首次通话结束关闭流后，后续音频无法写入，也无法支持多通话场景

修复代码 #

#!/usr/bin/env node
const WebSocket = require("ws");
const fs = require("fs");
const mulaw = require("mulaw-js");
const base64 = require("js-base64");
const express = require("express");
const app = express();
const server = require("http").createServer(app);
const wss = new WebSocket.Server({ server, port: 8080 });
const subscriptionKey = "[AZURE_SUBSCRIPTION_KEY_HERE]";
const serviceRegion = "eastus";
const language = "en-US";

const sdk = require("microsoft-cognitiveservices-speech-sdk");
// 公共配置提取，无需每个连接重复创建
const speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
speechConfig.speechRecognitionLanguage = language;

console.log("Listening at Port 8080");

// 每个WebSocket连接对应一路通话，单独创建识别实例
wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");
    // 每个通话单独创建推送流和识别器
    const azurePusher = sdk.AudioInputStream.createPushStream(sdk.AudioStreamFormat.getWaveFormatPCM(8000, 16, 1));
    const audioConfig = sdk.AudioConfig.fromStreamInput(azurePusher);
    const recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);

    // 识别事件监听移到连接内部
    recognizer.recognizing = (s, e) => {
        console.log(`RECOGNIZING: Text=${e.result.text}`);
    };

    recognizer.recognized = (s, e) => {
        if (e.result.reason == sdk.ResultReason.RecognizedSpeech) {
            console.log(`RECOGNIZED: Text=${e.result.text}`);
        } else if (e.result.reason == sdk.ResultReason.NoMatch) {
            console.log("NOMATCH: Speech could not be recognized.");
        }
    };

    recognizer.canceled = (s, e) => {
        console.log(`CANCELED: Reason=${e.reason}`);
        if (e.reason == sdk.CancellationReason.Error) {
            console.log(`CANCELED: ErrorCode=${e.errorCode}`);
            console.log(`CANCELED: ErrorDetails=${e.errorDetails}`);
        }
        recognizer.stopContinuousRecognitionAsync();
    };

    recognizer.sessionStopped = (s, e) => {
        console.log("\nSession stopped event.");
        recognizer.stopContinuousRecognitionAsync();
        recognizer.close();
    };

    // 启动当前通话的连续识别
    recognizer.startContinuousRecognitionAsync(() => {
        console.log("Continuous Reco Started");
    }, err => {
        console.trace("err - " + err);
        recognizer.close();
    });

    ws.on("message", function incoming(message) {
        const msg = JSON.parse(message);
        switch (msg.event) {
            case "connected":
                console.log(`A new call has connected.`);
                break;
            case "start":
                console.log(`Starting Media Stream ${msg.streamSid}`);
                break;
            case "media":
                process.stdout.write(`Receiving audio chunks\033[0G`);
                const streampayload = base64.decode(msg.media.payload);
                const data = Buffer.from(streampayload);
                // 转换为Azure要求的小端16位PCM Buffer
                const pcmSamples = mulaw.decode(data);
                const pcmBuffer = Buffer.alloc(pcmSamples.length * 2);
                for (let i = 0; i < pcmSamples.length; i++) {
                    pcmBuffer.writeInt16LE(pcmSamples[i], i * 2);
                }
                azurePusher.write(pcmBuffer);
                break;
            case "stop":
                console.log(`Call Has Ended`);
                azurePusher.close();
                recognizer.stopContinuousRecognitionAsync();
                break;
        }
    });

    // 处理连接异常断开的资源释放
    ws.on("close", () => {
        azurePusher.close();
        recognizer.stopContinuousRecognitionAsync(() => {
            recognizer.close();
        });
    });
});

内容的提问来源于stack exchange，提问作者Ken Arakelian