如何实现Google STT实时精准输出语音识别中间结果？

我想用Google STT和NAudio实现类似Google语音搜索的实时语音识别效果——说出内容时即时显示正确的中间文字，而非现在这样输出大量错误、重复的 interim 结果。目前只有最终识别结果正常，中间过程的输出混乱。

原实现代码 #

static async Task Main(string[] args)
{
    string credentialsPath = "D:/Docs/GoogleSttFile.json";
    System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath);
    string languageCode = "en-US";

    var speechClient = SpeechClient.Create();

    // Initialize audio input stream
    var audioStream = new WaveInEvent();
    audioStream.WaveFormat = new WaveFormat(16000, 16, 1); // Adjust format as needed

    var streamingCall = speechClient.StreamingRecognize();

    // Start streaming with the configuration
    var streamingConfig = new StreamingRecognitionConfig
    {
        Config = new RecognitionConfig
        {
            Encoding = RecognitionConfig.Types.AudioEncoding.Linear16,
            SampleRateHertz = 16000,
            LanguageCode = languageCode,
            EnableAutomaticPunctuation = true, // Enable punctuation
        },
        InterimResults = true, // Enable interim results
    };
    streamingCall.WriteAsync(new StreamingRecognizeRequest
    {
        StreamingConfig = streamingConfig,
    });

    var interimResultBuilder = new StringBuilder();
    string lastInterimResult = string.Empty;

    // Handle audio data
    audioStream.DataAvailable += (s, e) =>
    {
        if (e.Buffer.Length > 0)
        {
            streamingCall.WriteAsync(new StreamingRecognizeRequest
            {
                AudioContent = Google.Protobuf.ByteString.CopyFrom(e.Buffer, 0, e.BytesRecorded),
            });
        }
    };

    audioStream.StartRecording();

    Console.WriteLine("Listening...");

    // Start a new thread to listen for responses from the server
    var responseThread = new Thread(async () =>
    {
        await foreach (var response in streamingCall.GetResponseStream())
        {
            foreach (var result in response.Results)
            {
                string transcript = result.Alternatives[0].Transcript;
                if (result.IsFinal)
                {
                    Console.WriteLine(transcript);
                    interimResultBuilder.Clear();
                }
                else if (IsSignificantChange(transcript, lastInterimResult))
                {
                    interimResultBuilder.Append(transcript);
                    // Split interim results into complete sentences based on punctuation
                    var interimSentences = interimResultBuilder.ToString().Split(". ", StringSplitOptions.RemoveEmptyEntries);
                    foreach (var sentence in interimSentences)
                    {
                        Console.WriteLine("Interim result: " + sentence + ".");
                    }
                    interimResultBuilder.Clear();
                }

                lastInterimResult = transcript;
            }
        }
    });
    responseThread.Start();

    Console.WriteLine("Press Enter to stop recording...");
    Console.ReadLine();

    // Stop the audio recording and finish the streaming call
    audioStream.StopRecording();
    streamingCall.WriteCompleteAsync();
    responseThread.Join();

    Console.WriteLine("Press any key to exit...");
    Console.ReadKey();
}

static bool IsSignificantChange(string currentInterimResult, string lastInterimResult)
{
    // Customize this function to determine what constitutes a significant change
    // Here, we check if the current result is different from the last result.
    return !string.Equals(currentInterimResult, lastInterimResult, StringComparison.OrdinalIgnoreCase);
}

当前错误输出（说出"hello good morning"时） #

Interim result: how.
Interim result: hell.
Interim result: Hello..
Interim result: hello would.
Interim result: Hello.
Interim result: Good morning..
Interim result: Hello..
Interim result:  Good morning..
Interim result: Hello.
Interim result: Good morning..
Hello. Good morning.

核心问题分析 #

1. 错误的结果拼接逻辑：原代码用interimResultBuilder追加每次返回的transcript，导致重复叠加，拆分句子的方式完全错误。
2. 过度敏感的变化检测：只要和上一次结果有差异就输出，导致低置信度的错误结果频繁弹出。
3. 未模拟实时更新效果：Google语音搜索是覆盖当前输入行的内容，而非每次换行输出新结果。

修改后的代码 #

static async Task Main(string[] args)
{
    string credentialsPath = "D:/Docs/GoogleSttFile.json";
    System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialsPath);
    string languageCode = "en-US";

    var speechClient = SpeechClient.Create();
    var audioStream = new WaveInEvent();
    audioStream.WaveFormat = new WaveFormat(16000, 16, 1);

    var streamingCall = speechClient.StreamingRecognize();
    await streamingCall.WriteAsync(new StreamingRecognizeRequest
    {
        StreamingConfig = new StreamingRecognitionConfig
        {
            Config = new RecognitionConfig
            {
                Encoding = RecognitionConfig.Types.AudioEncoding.Linear16,
                SampleRateHertz = 16000,
                LanguageCode = languageCode,
                EnableAutomaticPunctuation = true,
                // 选用针对实时指令优化的模型
                Model = "command_and_search"
            },
            InterimResults = true,
            // 添加常用短语提升识别精度（可选）
            SpeechContexts = { new SpeechContext { Phrases = { "hello", "good morning" } } }
        }
    });

    audioStream.DataAvailable += (s, e) =>
    {
        if (e.BytesRecorded > 0)
        {
            _ = streamingCall.WriteAsync(new StreamingRecognizeRequest
            {
                AudioContent = Google.Protobuf.ByteString.CopyFrom(e.Buffer, 0, e.BytesRecorded)
            });
        }
    };

    audioStream.StartRecording();
    Console.WriteLine("Listening...");
    Console.WriteLine("Press Enter to stop recording...");

    string currentInterim = string.Empty;
    await foreach (var response in streamingCall.GetResponseStream())
    {
        foreach (var result in response.Results)
        {
            var alternative = result.Alternatives[0];
            // 过滤低置信度的中间结果（置信度范围0-1）
            if (!result.IsFinal && alternative.Confidence < 0.7)
                continue;

            string transcript = alternative.Transcript.Trim();
            if (result.IsFinal)
            {
                // 清除临时内容，输出最终结果
                Console.Write("\r" + new string(' ', Console.WindowWidth) + "\r");
                Console.WriteLine(transcript);
                currentInterim = string.Empty;
            }
            else
            {
                // 仅在结果有实质性变化时更新
                if (transcript != currentInterim)
                {
                    currentInterim = transcript;
                    // 覆盖当前行，模拟实时更新效果
                    Console.Write("\rInterim: " + currentInterim.PadRight(Console.WindowWidth - 9));
                }
            }
        }
    }

    audioStream.StopRecording();
    await streamingCall.WriteCompleteAsync();

    Console.WriteLine("\nPress any key to exit...");
    Console.ReadKey();
}

关键修改说明 #

• 选用command_and_search模型：该模型专门针对短指令、搜索类语音优化，实时识别精度更高。
• 置信度过滤：只处理置信度≥0.7的中间结果，过滤低精度错误识别。
• 实时覆盖输出：用Console.Write("\r")回到行首覆盖之前的内容，还原Google语音搜索的实时更新体验。
• 移除错误拼接逻辑：直接使用每次返回的transcript，避免重复叠加。
• 可选自定义短语：通过SpeechContexts添加常用短语，提升特定内容的识别准确率。

内容的提问来源于stack exchange，提问作者Pritom Routh