Swift对接ElevenLabs WebSocket流式音频播放异常及报错排查

问题现象 #

通过Swift与ElevenLabs建立WebSocket连接，传入文本接收流式音频分片，期望收到首个分片后立即连续播放，但实际仅播放少量内容就停止，同时出现两个报错：

WebSocket receive error: Error Domain=NSPOSIXErrorDomain Code=57 "Socket is not connected"
Audio player error: Error Domain=NSOSStatusErrorDomain Code=1954115647 "(null)"

相关代码 #

class WebSocketViewModel: NSObject, ObservableObject, URLSessionWebSocketDelegate {
    private var webSocketTask: URLSessionWebSocketTask?
    private var player: AVAudioPlayer?
    private var audioBuffer = Data()

    func connectAndPlay() {
        let urlString = "wss://api.elevenlabs.io/v1/text-to-speech/voice-id/stream-input?model_id=eleven_turbo_v2"
           guard let url = URL(string: urlString) else {
               print("Invalid URL")
               return
           }
           
           var request = URLRequest(url: url)

        let session = URLSession(configuration: .default, delegate: self, delegateQueue: OperationQueue())
        webSocketTask = session.webSocketTask(with: request)
        webSocketTask?.resume()
        sendInitialMessages(apiKey: apiKey)
    }

    private func sendInitialMessages(apiKey: String) {
        let bosMessage: [String: Any] = [
            "text": " ",
            "voice_settings": [
                "stability": 0.5,
                "similarity_boost": 0.8
            ],
            "xi_api_key": apiKey
        ]

        sendMessage(bosMessage)

        let textMessage: [String: Any] = [
            "text": "Hello World this is just a random message for testing purposes",
        ]

        sendMessage(textMessage)

        let eosMessage: [String: Any] = [
            "text": ""
        ]

        sendMessage(eosMessage)
    }

    private func sendMessage(_ message: [String: Any]) {
        do {
            let messageData = try JSONSerialization.data(withJSONObject: message, options: [])
            let messageString = String(data: messageData, encoding: .utf8) ?? ""
            let webSocketMessage = URLSessionWebSocketTask.Message.string(messageString)
            webSocketTask?.send(webSocketMessage) { error in
                if let error = error {
                    print("WebSocket send error: \(error)")
                }
            }
        } catch {
            print("JSON serialization error: \(error)")
        }
    }


    func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol `protocol`: String?) {
        print("WebSocket connection opened")
        receiveMessages()
    }

    func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) {
        print("WebSocket connection closed")
    }

    private func receiveMessages() {
        webSocketTask?.receive { [weak self] result in
            switch result {
            case .failure(let error):
                print("WebSocket receive error: \(error)")
            case .success(let message):
                switch message {
                case .string(let text):
                    self?.handleMessage(text)
                default:
                    break
                }
                self?.receiveMessages()
            }
        }
    }

    private func handleMessage(_ text: String) {
        guard let data = text.data(using: .utf8),
              let response = try? JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return }

        if let audioBase64 = response["audio"] as? String {
                    addToAudioBuffer(audioBase64)
                    if player == nil || !player!.isPlaying {
                        playAudio()
                    }
                }

        if let isFinal = response["isFinal"] as? Bool, isFinal {
            print("Audio generation is complete.")
        }

    }
    
    private func addToAudioBuffer(_ base64String: String) {
           guard let audioData = Data(base64Encoded: base64String) else {
               print("Failed to decode base64 audio data")
               return
           }
           audioBuffer.append(audioData)
       }

    private func playAudio() {
        guard !audioBuffer.isEmpty else {
                    print("Audio buffer is empty")
                    return
                }

                do {
                    player = try AVAudioPlayer(data: audioBuffer)
                    player?.play()
                } catch {
                    print("Audio player error: \(error)")
                }

                audioBuffer.removeAll()
    }
}

问题根源 #

1. Socket连接错误（Code57）：在WebSocket未完全建立连接时就发送消息，导致连接被强制中断。
2. 音频播放失败（Code1954115647）：AVAudioPlayer不支持流式追加音频数据，每次创建新实例并清空缓冲区的逻辑，会导致后续音频分片无法被播放，且旧的播放实例被覆盖后直接终止。
3. 消息发送顺序混乱：连续异步发送WebSocket消息，可能导致消息顺序错误，触发ElevenLabs服务端的异常处理，提前关闭连接。

修复方案 #

1. 延迟消息发送时机 #

将消息发送逻辑移到WebSocket连接成功的回调中，确保连接完全建立后再发送数据：

func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol `protocol`: String?) {
    print("WebSocket connection opened")
    sendInitialMessages(apiKey: apiKey) // 移至此处
    receiveMessages()
}

func connectAndPlay() {
    let urlString = "wss://api.elevenlabs.io/v1/text-to-speech/voice-id/stream-input?model_id=eleven_turbo_v2"
    guard let url = URL(string: urlString) else {
        print("Invalid URL")
        return
    }
    
    var request = URLRequest(url: url)

    let session = URLSession(configuration: .default, delegate: self, delegateQueue: OperationQueue())
    webSocketTask = session.webSocketTask(with: request)
    webSocketTask?.resume()
    // 移除原来的sendInitialMessages调用
}

2. 改用AVAudioEngine实现流式播放 #

替换AVAudioPlayer为AVAudioEngine，支持实时追加音频分片：

private var audioEngine: AVAudioEngine?
private var audioPlayerNode: AVAudioPlayerNode?
private var audioFormat: AVAudioFormat?

// 初始化音频引擎
private func setupAudioEngine() {
    audioEngine = AVAudioEngine()
    audioPlayerNode = AVAudioPlayerNode()
    audioEngine?.attach(audioPlayerNode!)
    
    // ElevenLabs返回的音频为PCM 16bit 44.1kHz单声道，匹配格式
    let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 44100.0, channels: 1, interleaved: false)
    audioFormat = format
    
    audioEngine?.connect(audioPlayerNode!, to: audioEngine!.mainMixerNode, format: format)
    
    do {
        try audioEngine?.start()
    } catch {
        print("音频引擎启动失败: \(error)")
    }
}

// 修改消息处理逻辑，直接流式播放
private func handleMessage(_ text: String) {
    guard let data = text.data(using: .utf8),
          let response = try? JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] else { return }

    if let audioBase64 = response["audio"] as? String {
        guard let audioData = Data(base64Encoded: audioBase64) else {
            print("Base64音频解码失败")
            return
        }
        playStreamingAudio(audioData: audioData)
    }

    if let isFinal = response["isFinal"] as? Bool, isFinal {
        print("音频生成完成")
    }
}

// 流式播放音频分片
private func playStreamingAudio(audioData: Data) {
    guard let format = audioFormat, let playerNode = audioPlayerNode else {
        print("音频引擎未初始化")
        return
    }
    
    let frameCount = AVAudioFrameCount(audioData.count) / format.streamDescription.pointee.mBytesPerFrame
    guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else {
        print("创建音频缓冲区失败")
        return
    }
    buffer.frameLength = frameCount
    audioData.copyBytes(to: buffer.mutableAudioBufferList.pointee.mBuffers.mData!, count: Int(buffer.mutableAudioBufferList.pointee.mBuffers.mDataByteSize))
    
    if !playerNode.isPlaying {
        playerNode.play()
    }
    playerNode.scheduleBuffer(buffer)
}

记得在connectAndPlay中调用setupAudioEngine()。

3. 保证消息发送顺序 #

修改消息发送方法，添加完成回调，实现链式发送，避免异步发送导致的顺序混乱：

private func sendInitialMessages(apiKey: String) {
    let bosMessage: [String: Any] = [
        "text": " ",
        "voice_settings": [
            "stability": 0.5,
            "similarity_boost": 0.8
        ],
        "xi_api_key": apiKey
    ]
    
    sendMessage(bosMessage) { [weak self] in
        let textMessage: [String: Any] = [
            "text": "Hello World this is just a random message for testing purposes",
        ]
        self?.sendMessage(textMessage) { [weak self] in
            let eosMessage: [String: Any] = [
                "text": ""
            ]
            self?.sendMessage(eosMessage)
        }
    }
}

private func sendMessage(_ message: [String: Any], completion: (() -> Void)? = nil) {
    do {
        let messageData = try JSONSerialization.data(withJSONObject: message, options: [])
        let messageString = String(data: messageData, encoding: .utf8) ?? ""
        let webSocketMessage = URLSessionWebSocketTask.Message.string(messageString)
        webSocketTask?.send(webSocketMessage) { error in
            if let error = error {
                print("WebSocket发送错误: \(error)")
            }
            completion?()
        }
    } catch {
        print("JSON序列化错误: \(error)")
        completion?()
    }
}

4. 完善连接关闭后的资源清理 #

在WebSocket断开时清理音频资源和连接任务：

func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) {
    let reasonStr = String(data: reason ?? Data(), encoding: .utf8) ?? "无"
    print("WebSocket连接关闭，代码: \(closeCode)，原因: \(reasonStr)")
    webSocketTask.cancel()
    self.webSocketTask = nil
    
    audioPlayerNode?.stop()
    audioEngine?.stop()
}

内容的提问来源于stack exchange，提问作者Sahil Singh