PyAudio音频流经WebSocket传输至网页JS播放的报错解决咨询

问题根源 #

decodeAudioData 是用于解码**完整的编码音频文件（如MP3、WAV）**的API，无法处理WebSocket传输的原始PCM音频流，这就是你报错的核心原因。要播放实时音频流，得用Web Audio API直接处理裸PCM数据。

解决方案步骤 #

1. 确保Python端输出标准PCM格式 #

先统一前后端的音频参数，确保pyaudiowpatch输出的是标准16位PCM数据，示例代码：

import pyaudiowpatch as pyaudio
import asyncio
import websockets

# 前后端必须一致的音频参数
FORMAT = pyaudio.paInt16  # 16位整数PCM
CHANNELS = 2  # 立体声，可改为1单声道
RATE = 44100  # 采样率
CHUNK = 1024  # 每次传输的帧数量

async def audio_server(websocket):
    with pyaudio.PyAudio() as p:
        # 获取系统内录设备的音频流
        with p.open(format=FORMAT,
                    channels=CHANNELS,
                    rate=RATE,
                    input=True,
                    input_device_index=p.get_default_loopback_device_info()["index"],
                    frames_per_buffer=CHUNK) as stream:
            while True:
                data = stream.read(CHUNK)
                await websocket.send(data)

async def main():
    async with websockets.serve(audio_server, "localhost", 8765):
        await asyncio.Future()  # 保持服务运行

if __name__ == "__main__":
    asyncio.run(main())

关键：必须使用paInt16格式，且采样率、声道数、帧大小参数要和前端完全匹配。

2. 前端JS用Web Audio API处理实时PCM流 #

用AudioContext配合ScriptProcessorNode（兼容性好）或AudioWorklet（现代高性能方案）直接处理PCM数据，以下是两种实现方式：

方案一：ScriptProcessorNode（兼容多数浏览器）

const audioContext = new (window.AudioContext || window.webkitAudioContext)({
    sampleRate: 44100  // 和Python端RATE一致
});
// 创建音频处理节点，参数：缓冲区大小、输入声道数、输出声道数
const scriptNode = audioContext.createScriptProcessor(1024, 2, 2);

scriptNode.onaudioprocess = function(event) {
    const outputBuffer = event.outputBuffer;
    if (audioBufferQueue.length === 0) {
        // 无数据时输出静音
        for (let channel = 0; channel < outputBuffer.numberOfChannels; channel++) {
            outputBuffer.getChannelData(channel).fill(0);
        }
        return;
    }

    const inputData = audioBufferQueue.shift();
    for (let channel = 0; channel < outputBuffer.numberOfChannels; channel++) {
        const outputData = outputBuffer.getChannelData(channel);
        // 将16位PCM转换为Web Audio需要的-1~1浮点值
        for (let i = 0; i < outputData.length; i++) {
            const offset = (i * outputBuffer.numberOfChannels + channel) * 2;
            if (offset + 1 < inputData.length) {
                // 合并两个字节为16位整数
                const sample = inputData.getInt16(offset, true);
                outputData[i] = sample / 32768;
            } else {
                outputData[i] = 0;
            }
        }
    }
};

scriptNode.connect(audioContext.destination);

// 缓存音频数据的队列
const audioBufferQueue = [];

// 建立WebSocket连接
const ws = new WebSocket('ws://localhost:8765');
ws.binaryType = 'arraybuffer';  // 必须设置为arraybuffer接收二进制数据

ws.onmessage = function(event) {
    audioBufferQueue.push(new DataView(event.data));
};

// 浏览器要求用户交互才能激活音频上下文
document.addEventListener('click', () => {
    if (audioContext.state === 'suspended') {
        audioContext.resume();
    }
});

方案二：AudioWorklet（现代高性能方案）

1. 新建audio-worklet-processor.js文件：

class AudioStreamProcessor extends AudioWorkletProcessor {
    constructor() {
        super();
        this.bufferQueue = [];
        this.port.onmessage = (event) => {
            this.bufferQueue.push(new DataView(event.data));
        };
    }

    process(inputs, outputs) {
        const output = outputs[0];
        const channelCount = output.length;
        const frameCount = output[0].length;

        if (this.bufferQueue.length === 0) {
            for (let channel = 0; channel < channelCount; channel++) {
                output[channel].fill(0);
            }
            return true;
        }

        const inputBuffer = this.bufferQueue[0];
        let bytesUsed = 0;

        for (let channel = 0; channel < channelCount; channel++) {
            const channelData = output[channel];
            for (let i = 0; i < frameCount; i++) {
                const offset = (i * channelCount + channel) * 2;
                if (offset + 1 < inputBuffer.byteLength) {
                    const sample = inputBuffer.getInt16(offset, true);
                    channelData[i] = sample / 32768;
                    bytesUsed += 2;
                } else {
                    channelData[i] = 0;
                }
            }
        }

        // 处理剩余未用完的缓冲区数据
        if (bytesUsed >= inputBuffer.byteLength) {
            this.bufferQueue.shift();
        } else {
            this.bufferQueue[0] = new DataView(inputBuffer.buffer, bytesUsed);
        }

        return true;
    }
}

registerProcessor('audio-stream-processor', AudioStreamProcessor);

2. 前端主JS代码：

async function initAudioStream() {
    const audioContext = new (window.AudioContext || window.webkitAudioContext)({
        sampleRate: 44100
    });

    // 加载AudioWorklet处理器
    await audioContext.audioWorklet.addModule('audio-worklet-processor.js');
    const workletNode = new AudioWorkletNode(audioContext, 'audio-stream-processor');
    workletNode.connect(audioContext.destination);

    const ws = new WebSocket('ws://localhost:8765');
    ws.binaryType = 'arraybuffer';

    ws.onmessage = (event) => {
        workletNode.port.postMessage(event.data);
    };

    // 激活音频上下文
    document.addEventListener('click', () => {
        if (audioContext.state === 'suspended') {
            audioContext.resume();
        }
    });
}

initAudioStream();

内容的提问来源于stack exchange，提问作者Nick