Django WebSocket+React接入Google Speech API无转写无报错问题求助

尝试在Django+React项目中集成Google Speech API实现语音交互功能，实现流程为React前端通过mediaDevices采集用户语音，将音频流转为buffer后通过WebSocket发送到Django后端，Django侧将buffer转写为文本后返回给React客户端。目前遇到的问题是Google API既不返回转写结果，也不抛出任何错误，仅返回<google.api_core.grpc_helpers._StreamingResponseIterator object at 0x000001CD1D2C5E88>对象。
根据官方文档说明该返回值为可迭代对象，但无法对该对象执行遍历操作。

React代码 #

import React, { Component } from 'react'
import * as audioUtils from "./audioUtils";
import mic from "microphone-stream";
import "./style.scss";


class VttMic extends Component {
    state = {}
    transcription = []
    SAMPLE_RATE = 16000;
    SAMPLE_SIZE = 16
    
    startTranscribe = () => {
        window.navigator.mediaDevices.getUserMedia({
            video: false,
            audio: {
                echoCancellation: true,
                channelCount: 1,
                sampleRate: {
                    ideal: this.SAMPLE_RATE
                },
                sampleSize: this.SAMPLE_SIZE
            }
        })
        .then(this.streamAudioToWebSocket) 
        .catch( (error) => {
            this.handleError("There was an error streaming your audio to Google Speech. Please try again.")
        });
    }

    streamAudioToWebSocket = async (userMediaStream) => {
        try{
            this.micStream = new mic();
            this.micStream.on("format", (data) => {
                this.inputSampleRate = data.sampleRate;
            });
            this.micStream.setStream(userMediaStream);
            this.socket = new WebSocket('ws://localhost:8000/ws/stt/');
            this.socket.binaryType = "arraybuffer";
            this.socket.onopen = () => {
                this.micStream.on('data', (rawAudioChunk) => {
                    let binary = this.convertAudioToBinaryMessage(rawAudioChunk);
                    if (this.socket.readyState === this.socket.OPEN)
                        this.socket.send(binary);
                }
            )};
            this.wireSocketEvents();
        }catch(err){
            this.handleError("An error occured at streamAudioToWebSocket", err);
        }
    }


    wireSocketEvents() {
        let transcribeException = false;
        let socketError = false;
        this.socket.onmessage = (message) => {
        };
    
        this.socket.onerror = () => {
            socketError = true;
            this.handleError("WebSocket connection error. Try again.");
        };
        
        this.socket.onclose = (closeEvent) => {
            this.micStream.stop();
            if (!socketError && !transcribeException) {
                if (closeEvent.code != 1000) {
                    this.handleError('Streaming Exception: ' + closeEvent.reason);
                }
            }
        };
    }

    convertAudioToBinaryMessage(audioChunk) {
        let raw = mic.toRaw(audioChunk);
        if (raw == null)
            return;    
        let downsampledBuffer = audioUtils.downsampleBuffer(raw, this.inputSampleRate, this.SAMPLE_RATE);
        let pcmEncodedBuffer = audioUtils.pcmEncode(downsampledBuffer);    
        return pcmEncodedBuffer;
    }

    closeSocket = () => {
        if (this.socket && this.socket.readyState === this.socket.OPEN) {
            this.micStream.stop();
            this.socket.close();
        }
    }

    handleError = (err) => {
        this.closeTranscription();
        this.props.onError && this.props.onError(err);
    }

    handleMicClick = () => {
        if(this.state.started){
            this.closeTranscription();
        }else{
            this.startTranscription();
        }
    }

    startTranscription = () => {
        this.setState({started: true});
        this.startTranscribe();
    }

    closeTranscription = () => {
        this.setState({started: false});
        this.transcription = [];
        this.closeSocket();
        this.props.onClose && this.props.onClose()
    }

    startCloseWaitTimer = () => {
        if(this.t)
            clearTimeout(this.t);
        this.t = setTimeout(() => {
            this.closeTranscription();
        }, 10000);
    }
    
    render() {
        let { started } = this.state;
        return (
            <div className="stt-mic-container" onClick={this.handleMicClick}>
                <i 
                    className={`sttMic fa fa-microphone text-primary`} 
                />
                <div class={`circle-ani ${started ? "active" : ""}`} />
            </div>
        )
    }
}

export default VttMic;

React audioUtils.js代码 #

export function pcmEncode(input) {
    var offset = 0;
    var buffer = new ArrayBuffer(input.length * 2);
    var view = new DataView(buffer);
    for (var i = 0; i < input.length; i++, offset += 2) {
        var s = Math.max(-1, Math.min(1, input[i]));
        view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
    }
    return buffer;
}

export function downsampleBuffer(buffer, inputSampleRate = 44100, outputSampleRate = 16000) {
        
    if (outputSampleRate === inputSampleRate) {
        return buffer;
    }

    var sampleRateRatio = inputSampleRate / outputSampleRate;
    var newLength = Math.round(buffer.length / sampleRateRatio);
    var result = new Float32Array(newLength);
    var offsetResult = 0;
    var offsetBuffer = 0;
    
    while (offsetResult < result.length) {

        var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);

        var accum = 0,
        count = 0;
        
        for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++ ) {
            accum += buffer[i];
            count++;
        }

        result[offsetResult] = accum / count;
        offsetResult++;
        offsetBuffer = nextOffsetBuffer;

    }

    return result;

}

Django Channels代码 #

import json
from channels.generic.websocket import WebsocketConsumer
from google.cloud import speech
from google.oauth2 import service_account
credentials = service_account.Credentials.from_service_account_file("319006-603d31ac86ca.json")
client = speech.SpeechClient(credentials=credentials)

class SpeechToTextConsumer(WebsocketConsumer):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.stream = []
        self.streaming_config = None
        self.language_code = "zh-CN" # 按需替换为对应语言编码，英文为en-US

    def connect(self):
        self.accept()
        rate = 16000
        config = speech.RecognitionConfig(
            encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
            sample_rate_hertz=rate,
            language_code=self.language_code,
        )
        self.streaming_config = speech.StreamingRecognitionConfig(
            config=config, interim_results=True
        )

    def disconnect(self, close_code):
        print("disconnect")

    def process(self, responses):
        print(responses) 
        for response in responses:
            print("response", response)
            if not response.results:
                print("continue")
                continue
            result = response.results[0]
            print("result", result)
            self.send(text_data=json.dumps({"transcript": result.alternatives[0].transcript}))
    
    def receive(self, text_data=None, bytes_data=None):
        self.stream.append(bytes_data)
        # 累积够足够的音频块再发起识别，或修改为持续流式推送逻辑
        if len(self.stream) >= 10:
            requests = [speech.StreamingRecognizeRequest(streaming_config=self.streaming_config)] + [
                speech.StreamingRecognizeRequest(audio_content=chunk) for chunk in self.stream
            ]
            responses = client.streaming_recognize(self.streaming_config, iter(requests))
            self.process(responses)
            self.stream = []

• 修正Django代码中的语法与逻辑错误：
  1. 补充未定义的language_code变量、json依赖导入，删除重复初始化的SpeechClient实例
  2. Google Speech流式识别要求第一个请求必须携带配置信息，原代码单独声明了配置请求但未加入请求序列，这是无返回的核心原因
  3. 原逻辑每收到一个音频chunk就发起一次新请求，单个音频长度过短无法触发识别，需要累积一定长度的音频块或维护持续的请求流
• 修正React端的错误：
  1. 音频降采样时采样率参数大小写错误，原代码写的this.sampleRate应为this.SAMPLE_RATE，导致降采样逻辑异常，音频格式不符合要求
  2. 错误提示文案笔误，原代码中错误提示为Amazon Transcribe，可按需修改为对应服务名称
• 阻塞问题处理：grpc的流式迭代器是阻塞对象，建议使用异步线程或Django Channels的异步Consumer处理遍历逻辑，避免阻塞WebSocket消息接收
• 调试优化：可先调用Google Speech同步识别接口测试单个音频文件的识别效果，排除音频格式、鉴权等基础问题后再调试流式逻辑

内容的提问来源于stack exchange，提问作者Lalit Sharma