Azure Speech SDK ConversationTranscriber 前后端音频流转配置问题

针对你遇到的前端MediaRecorder输出格式与Azure ConversationTranscriber不兼容的问题，以下是两种高效的解决思路，无需依赖FFmpeg实时转码：

一、优先方案：WebM/OPUS → 提取OPUS裸流直接适配OGG/OPUS格式 #

Azure认知语音服务原生支持OGG容器的OPUS编码，而前端MediaRecorder生成的WebM/OPUS只是容器不同，编码本身完全一致。只需在后端解析WebM分片，提取OPUS裸流即可直接发送给服务。

前端代码（采集并发送WebM/OPUS分片） #

// 强制单声道、16kHz采样率（Azure兼容参数）
const constraints = {
  audio: {
    channelCount: 1,
    sampleRate: 16000,
    echoCancellation: true
  }
};

const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
// 指定WebM/OPUS格式
const recorder = new MediaRecorder(mediaStream, { mimeType: 'audio/webm;codecs=opus' });
// 每100ms发送一次音频分片
recorder.start(100);

const webSocket = new WebSocket('ws://your-backend-url');

recorder.ondataavailable = (e) => {
  if (e.data.size > 0 && webSocket.readyState === WebSocket.OPEN) {
    webSocket.send(e.data);
  }
};

后端代码（解析WebM提取OPUS裸流） #

使用pywebm库解析WebM分片，提取音频帧后喂给ConversationTranscriber：

import io
import pywebm
from azure.cognitiveservices.speech import (
    AudioStreamFormat, PullAudioInputStream,
    PullAudioInputStreamCallback, SpeechConfig,
    ConversationTranscriber
)

# 配置Azure语音服务
speech_config = SpeechConfig(subscription="YOUR_AZURE_KEY", region="YOUR_REGION")
# 设置为OGG/OPUS格式（16kHz，单声道）
audio_format = AudioStreamFormat.get_ogg_opus_format(16000, 1)

class WebMOpusStream(PullAudioInputStreamCallback):
    def __init__(self, websocket):
        self.websocket = websocket
        self.opus_buffer = b""

    def read(self, buffer: memoryview) -> int:
        # 从WebSocket接收WebM数据
        if not self.opus_buffer:
            webm_data = self.websocket.recv()
            # 解析WebM，提取OPUS音频帧
            reader = pywebm.WebMReader(io.BytesIO(webm_data))
            for frame in reader.iter_frames():
                if frame.track_type == "audio":
                    self.opus_buffer += frame.data

        # 将OPUS数据写入输出buffer
        write_len = min(len(self.opus_buffer), len(buffer))
        buffer[:write_len] = self.opus_buffer[:write_len]
        self.opus_buffer = self.opus_buffer[write_len:]
        return write_len

# 初始化WebSocket连接（示例用websockets库）
import websockets
import asyncio

async def handle_client(websocket):
    stream_callback = WebMOpusStream(websocket)
    audio_stream = PullAudioInputStream(audio_format, stream_callback)
    transcriber = ConversationTranscriber(speech_config, audio_stream=audio_stream)

    # 绑定转录回调
    def on_transcribed(evt):
        if evt.result.text.strip():
            print(f"转录结果: {evt.result.text}")

    transcriber.transcribed.connect(on_transcribed)
    await transcriber.start_transcribing_async()

    # 保持连接
    await websocket.wait_closed()
    await transcriber.stop_transcribing_async()

# 启动WebSocket服务
start_server = websockets.serve(handle_client, "0.0.0.0", 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

二、备选方案：前端采集原生PCM → 后端实时适配WAV/PCM格式 #

如果不想处理WebM容器，直接用WebAudio API采集16位单声道16kHz的PCM裸流，后端只需按Azure要求的参数配置即可直接接收（无需额外添加WAV头，Azure支持指定参数的裸PCM）。

前端代码（采集原生PCM） #

const constraints = {
  audio: { channelCount: 1, sampleRate: 16000 }
};
const mediaStream = await navigator.mediaDevices.getUserMedia(constraints);
const audioContext = new AudioContext({ sampleRate: 16000 });
const source = audioContext.createMediaStreamSource(mediaStream);
// 4096帧的缓冲区，平衡延迟与性能
const processor = audioContext.createScriptProcessor(4096, 1, 1);

source.connect(processor);
processor.connect(audioContext.destination);

const webSocket = new WebSocket('ws://your-backend-url');

processor.onaudioprocess = (e) => {
  if (webSocket.readyState !== WebSocket.OPEN) return;
  
  // 获取Float32格式的PCM数据，转换为16位整数
  const float32Data = e.inputBuffer.getChannelData(0);
  const int16Data = new Int16Array(float32Data.length);
  for (let i = 0; i < float32Data.length; i++) {
    // 归一化到-32768到32767范围
    const sample = Math.max(-1, Math.min(1, float32Data[i]));
    int16Data[i] = sample < 0 ? sample * 0x8000 : sample * 0x7FFF;
  }
  
  webSocket.send(int16Data.buffer);
};

后端代码（处理PCM流） #

直接使用Azure的PCM格式配置，无需额外转码：

from azure.cognitiveservices.speech import (
    AudioStreamFormat, PullAudioInputStream,
    PullAudioInputStreamCallback, SpeechConfig,
    ConversationTranscriber
)
import websockets
import asyncio

speech_config = SpeechConfig(subscription="YOUR_AZURE_KEY", region="YOUR_REGION")
# 配置PCM格式：16kHz采样率，16位深度，单声道
audio_format = AudioStreamFormat.get_pcm_format(16000, 16, 1)

class PCMAudioStream(PullAudioInputStreamCallback):
    def __init__(self, websocket):
        self.websocket = websocket
        self.pcm_buffer = b""

    def read(self, buffer: memoryview) -> int:
        if not self.pcm_buffer:
            self.pcm_buffer = self.websocket.recv()
        
        write_len = min(len(self.pcm_buffer), len(buffer))
        buffer[:write_len] = self.pcm_buffer[:write_len]
        self.pcm_buffer = self.pcm_buffer[write_len:]
        return write_len

async def handle_client(websocket):
    stream_callback = PCMAudioStream(websocket)
    audio_stream = PullAudioInputStream(audio_format, stream_callback)
    transcriber = ConversationTranscriber(speech_config, audio_stream=audio_stream)

    def on_transcribed(evt):
        if evt.result.text.strip():
            print(f"转录结果: {evt.result.text}")

    transcriber.transcribed.connect(on_transcribed)
    await transcriber.start_transcribing_async()
    await websocket.wait_closed()
    await transcriber.stop_transcribing_async()

start_server = websockets.serve(handle_client, "0.0.0.0", 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

关键注意事项 #

• 参数一致性：前端采集和后端配置必须统一使用单声道、16kHz采样率、16位深度，这是Azure认知语音服务最稳定兼容的参数组合。
• 分片大小：前端发送的音频分片建议控制在100-200ms范围内，避免过大导致延迟，过小增加网络开销。
• 异常处理：需添加WebSocket断开、音频流中断等异常捕获逻辑，保证转录服务的稳定性。

内容的提问来源于stack exchange，提问作者MaciekT