Unity中API响应MP3音频流式播放无声音问题求解

问题现状 #

通过POST请求向API发送音频文件后，API返回可流式传输的MP3音频。目前仅能等待音频完全下载后播放，尝试流式接收分片数据并即时播放时无声音输出，现有核心代码如下：

请求发送与流式接收代码 #

private IEnumerator WaitAndSendRequest(byte[] audioData, string conversationId)
{ 
    request = new HTTPRequest(new Uri(serverUrl), HTTPMethods.Post, OnRequestFinished);
    request.SetHeader("Content-Type", "multipart/form-data");
    request.AddField("conversation_id", conversationId);
    request.AddBinaryData("audio_file", audioData, "audio_sample.mp3", "audio/mpeg");
    request.StreamChunksImmediately = true; 
    request.StreamFragmentSize = 1024;
    request.OnDownloadProgress = OnDownloadProgress;
    request.OnUploadProgress = OnUploadProgressDelegate;
    request.OnStreamingData = OnStreamingDataReceived;
    Debug.Log("Uploading audio...");
    request.Send();  
    yield return new WaitUntil(() => !isRequestInProgress); 
} 

private void OnDownloadProgress(HTTPRequest request, long downloaded, long downloadLength)
{ 
    float progress = (float)downloaded / downloadLength * 100f;
}

private void OnUploadProgressDelegate(HTTPRequest request, long downloaded, long downloadLength)
{ 
    float progress = (float)downloaded / downloadLength * 100f;
} 

private bool OnStreamingDataReceived(HTTPRequest request, HTTPResponse response, byte[] dataFragment, int dataFragmentLength)
{
    Debug.Log("dataFragmentLength: " + dataFragmentLength);
    StartCoroutine(PlayReceivedAudioChunk(dataFragment)); 
    return true;
}

private IEnumerator PlayReceivedAudioChunk(byte[] audioData)
{ 
    AudioClip audioClip = WavUtility.ToAudioClip(audioData, "AudioStream");
    if (audioClip != null)
    {
        audioSource.clip = audioClip;
        audioSource.Play();
        Debug.Log("Playing audio chunk." + audioData); 
        yield return new WaitUntil(() => !audioSource.isPlaying);
    }
    else
    {
        Debug.LogError("Failed to create audio clip from chunk.");
    }
}

private void OnRequestFinished(HTTPRequest req, HTTPResponse resp)
{
    try
    {
        if (resp == null || !resp.IsSuccess)
        { 
            Debug.LogError("Error uploading audio: " + (resp != null ? resp.Message : "Unknown error"));
            UIManager.Instance.connectionTxt.text = "An error occurred, please help me check the network connection!.";  
            return;
        }
        Debug.Log("Audio uploaded successfully. Processing response...");  
        byte[] responseAudioData = resp.Data;
        StartCoroutine(PlayReceivedAudio(responseAudioData));
    }
    finally
    { 
        isRequestInProgress = false;
    }  
}

private IEnumerator PlayReceivedAudio(byte[] audioData)
{ 
    string tempFilePath = Path.Combine(Application.temporaryCachePath, "processed_audio.mp3");
    File.WriteAllBytes(tempFilePath, audioData);
    using (UnityWebRequest www = UnityWebRequestMultimedia.GetAudioClip("file://" + tempFilePath, AudioType.MPEG))
    {
        yield return www.SendWebRequest();
        if (www.result == UnityWebRequest.Result.ConnectionError || www.result == UnityWebRequest.Result.ProtocolError)
        {
            Debug.LogError("Error loading audio: " + www.error);
            yield break;
        }
        AudioClip audioClip = DownloadHandlerAudioClip.GetContent(www);
        audioSource.clip = audioClip;
        audioSource.Play();
        Debug.Log("Playing received audio.");
        StartCoroutine(CheckAudioFinished());
    }
}

无效的音频转换函数 #

原代码中用于转换音频的ToAudioClip仅支持WAV格式的原始PCM数据，无法处理MP3编码数据：

public static AudioClip ToAudioClip(byte[] data, string name)
{ 
    int sampleCount = data.Length / 2;  
    float[] samples = new float[sampleCount];
    for (int i = 0; i < sampleCount; i++)
    {
        samples[i] = (short)(data[i * 2] | (data[i * 2 + 1] << 8)) / 32768.0f;  
    } 
    AudioClip audioClip = AudioClip.Create(name, sampleCount, 1, 44100, false);
    audioClip.SetData(samples, 0);
    return audioClip;
}

核心问题分析 #

1. 格式不匹配：ToAudioClip将MP3编码字节当作WAV原始PCM解析，导致音频数据完全错误，自然无声音。
2. 分片数据不完整：MP3以帧为单位编码，单个1024字节的分片可能无法构成完整可解码的MP3帧，解码会失败。
3. 播放逻辑冲突：每次替换AudioSource.clip会中断当前播放，无法实现连贯的流式播放。

可行解决方案 #

1. 定义全局缓存与状态变量 #

在脚本中添加以下全局变量，用于缓存MP3数据和管理播放状态：

private MemoryStream mp3Stream = new MemoryStream();
private bool isProcessingAudio = false;
private AudioSource audioSource;

2. 修改流式数据接收逻辑 #

不再直接播放分片，而是将数据累积到内存流中，尝试解码：

private bool OnStreamingDataReceived(HTTPRequest request, HTTPResponse response, byte[] dataFragment, int dataFragmentLength)
{
    // 将分片数据写入内存流缓存
    mp3Stream.Write(dataFragment, 0, dataFragmentLength);
    // 尝试解码当前累积的数据
    if (!isProcessingAudio)
    {
        StartCoroutine(TryDecodeAndPlayMp3());
    }
    return true;
}

3. 实现MP3解码与队列播放 #

利用Unity内置的MP3解码能力，将缓存的MP3数据转换为AudioClip，并实现连贯的队列播放：

private IEnumerator TryDecodeAndPlayMp3()
{
    isProcessingAudio = true;
    byte[] tempMp3Data = mp3Stream.ToArray();
    
    // 用Base64将内存中的MP3数据转为UnityWebRequest可识别的URI
    string base64Mp3 = Convert.ToBase64String(tempMp3Data);
    using (UnityWebRequest www = UnityWebRequestMultimedia.GetAudioClip($"data:audio/mpeg;base64,{base64Mp3}", AudioType.MPEG))
    {
        yield return www.SendWebRequest();
        
        if (www.result == UnityWebRequest.Result.Success)
        {
            AudioClip decodedClip = DownloadHandlerAudioClip.GetContent(www);
            if (decodedClip != null)
            {
                // 播放解码后的音频片段
                StartCoroutine(PlayAudioQueue(decodedClip));
                // 清空已解码的缓存数据，避免重复解码
                mp3Stream.SetLength(0);
            }
        }
        else
        {
            // 解码失败，说明数据不足，等待下一个分片
            Debug.Log("Waiting for more MP3 data to decode");
        }
    }
    isProcessingAudio = false;
    
    // 检查是否还有未处理的缓存数据
    if (mp3Stream.Length > 0 && !isProcessingAudio)
    {
        StartCoroutine(TryDecodeAndPlayMp3());
    }
}

private IEnumerator PlayAudioQueue(AudioClip clip)
{
    // 等待当前音频播放完毕再播放新片段
    if (audioSource.isPlaying)
    {
        yield return new WaitUntil(() => !audioSource.isPlaying);
    }
    
    audioSource.clip = clip;
    audioSource.Play();
    // 等待当前片段播放结束
    yield return new WaitUntil(() => !audioSource.isPlaying);
}

4. 处理请求结束后的剩余数据 #

在OnRequestFinished中处理最后一批缓存的MP3数据：

private void OnRequestFinished(HTTPRequest req, HTTPResponse resp)
{
    try
    {
        if (resp == null || !resp.IsSuccess)
        { 
            Debug.LogError("Error uploading audio: " + (resp != null ? resp.Message : "Unknown error"));
            UIManager.Instance.connectionTxt.text = "An error occurred, please help me check the network connection!.";  
            return;
        }
        
        // 处理请求结束后剩余的MP3数据
        if (mp3Stream.Length > 0 && !isProcessingAudio)
        {
            StartCoroutine(TryDecodeAndPlayMp3());
        }
    }
    finally
    { 
        isRequestInProgress = false;
    }  
}

5. 清理资源 #

在脚本销毁时释放内存流资源：

private void OnDestroy()
{
    mp3Stream?.Dispose();
}

关键优化点 #

• 数据缓存：通过MemoryStream累积MP3数据，确保有足够的字节构成可解码的MP3帧。
• 内置解码：利用UnityWebRequest的MP3解码能力，无需手动处理复杂的MP3帧结构。
• 队列播放：避免直接替换AudioClip导致的播放中断，实现连贯的流式音频输出。

内容的提问来源于stack exchange，提问作者nghien_rbc