iOS实时音频处理：如何实现1024采样帧无卡顿低延迟处理？

问题背景 #

• 需求：iOS平台实现麦克风采集→自定义函数处理→扬声器输出的实时音频流，要求处理缓冲区固定为1024采样帧
• 初始尝试：使用AVAudioEngine的installTap方案，即使无任何处理逻辑仍出现音频卡顿
• 改进尝试：改用AVAudioSinkNode对接AVAudioSourceNode实现了实时运行，但无法强制处理缓冲区大小为1024帧

初始installTap方案卡顿的原因分析 #

1. 音频路径冲突：同时连接了inputNode→mixerNode→outputNode原生播放路径，又通过tap采集inputNode并使用playerNode二次播放，导致音频时序混乱
2. 播放调度错误：scheduleBuffer使用.interrupts选项，每次调用都会中断当前正在播放的缓冲区，直接引发断流卡顿
3. 格式强制转换：强制指定双声道格式，但设备麦克风可能输出单声道，额外的格式转换增加了处理延迟

解决AVAudioSinkNode/AVAudioSourceNode固定1024帧缓冲区的方案 #

核心思路：通过帧累积+按需拆分的方式，在SinkNode中凑够1024帧再处理，在SourceNode中根据硬件要求拆分输出，既满足固定大小的处理需求，又适配硬件的缓冲区要求。

具体实现步骤 #

1. 添加帧累积缓冲区：维护一个临时缓冲区，用于累积SinkNode传入的零散帧，直到达到1024帧
2. 固定大小帧处理：当累积满1024帧时，调用自定义处理函数，将处理后的完整1024帧缓冲区存入循环队列
3. SourceNode按需输出：从循环队列取出1024帧缓冲区，根据硬件要求的frameCount拆分数据并输出，避免帧大小不匹配

修正后的完整代码 #

import SwiftUI
import AVFoundation
import Combine

class CircularAudioFrameQueue {
    private var queue: [AVAudioPCMBuffer]
    private var headIndex = 0
    private var tailIndex = 0
    private let maxSize: Int
    private let format: AVAudioFormat
    
    init(format: AVAudioFormat, size: Int = 10) {
        self.maxSize = size
        self.format = format
        self.queue = (0..<size).compactMap { _ in AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024) }
    }
    
    func enqueue(_ buffer: AVAudioPCMBuffer) {
        queue[headIndex] = buffer
        headIndex = (headIndex + 1) % maxSize
        if headIndex == tailIndex {
            tailIndex = (tailIndex + 1) % maxSize
        }
    }
    
    func dequeue() -> AVAudioPCMBuffer? {
        guard tailIndex != headIndex else { return nil }
        let buffer = queue[tailIndex]
        tailIndex = (tailIndex + 1) % maxSize
        return buffer
    }
}

class AudioRecorderManager: NSObject, ObservableObject {
    private var audioEngine = AVAudioEngine()
    private var noiseCanceller: ONNXNoiseCanceller?
    private var frameQueue: CircularAudioFrameQueue?
    
    // 帧累积缓冲区：用于凑够1024帧再处理
    private var accumulatedBuffer: AVAudioPCMBuffer?
    private let targetFrameCount: AVAudioFrameCount = 1024
    
    @Published var isRecording = false
    @Published var errorMessage: String?
    
    override init() {
        super.init()
        noiseCanceller = ONNXNoiseCanceller()
        setupAudioEngine()
    }
    
    private func setupAudioEngine() {
        do {
            let audioSession = AVAudioSession.sharedInstance()
            try audioSession.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker, .allowBluetooth])
            try audioSession.setActive(true)
        } catch {
            errorMessage = "Failed to set up audio session: \(error.localizedDescription)"
            print(errorMessage ?? "")
            return
        }
        
        let inputNode = audioEngine.inputNode
        let inputFormat = inputNode.outputFormat(forBus: 0)
        
        guard let processingFormat = AVAudioFormat(
            standardFormatWithSampleRate: 44100,
            channels: 2
        )else {
            errorMessage = "Failed to create audio format"
            print(errorMessage ?? "")
            return
        }
        frameQueue = CircularAudioFrameQueue(format: processingFormat)
        accumulatedBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: targetFrameCount)
        accumulatedBuffer?.frameLength = 0
        
        // SinkNode：采集输入并累积到1024帧后处理
        let sinkNode = AVAudioSinkNode { [weak self] (timestamp, frameCount, audioBufferList) -> OSStatus in
            guard let self = self,
                  let accumBuffer = self.accumulatedBuffer,
                  let processingFormat = self.frameQueue?.format else {
                return noErr
            }
            
            let ablPointer = UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: audioBufferList))
            let incomingBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: frameCount)!
            incomingBuffer.frameLength = frameCount
            
            // 拷贝输入数据到临时缓冲区
            for bufferIndex in 0..<min(ablPointer.count, Int(processingFormat.channelCount)) {
                let inBuffer = ablPointer[bufferIndex]
                let outBuffer = incomingBuffer.floatChannelData?[bufferIndex]
                if let inData = inBuffer.mData?.assumingMemoryBound(to: Float.self), let outData = outBuffer {
                    memcpy(outData, inData, Int(frameCount) * MemoryLayout<Float>.stride)
                }
            }
            
            // 累积帧到目标大小
            let availableFrames = accumBuffer.frameCapacity - accumBuffer.frameLength
            let framesToCopy = min(frameCount, availableFrames)
            
            if framesToCopy > 0 {
                accumBuffer.append(incomingBuffer, from: 0, frameCount: framesToCopy)
            }
            
            // 当累积满1024帧时，处理并入队
            if accumBuffer.frameLength == self.targetFrameCount {
                // 这里替换为你的自定义处理逻辑
                let processedBuffer = self.processAudioBuffer(accumBuffer)
                self.frameQueue?.enqueue(processedBuffer)
                
                // 重置累积缓冲区
                accumBuffer.frameLength = 0
            }
            
            return noErr
        }
        
        // SourceNode：从队列取帧并按需输出
        let sourceNode = AVAudioSourceNode { [weak self] (silence, timeStamp, frameCount, audioBufferList) -> OSStatus in
            guard let self = self,
                  let processingFormat = self.frameQueue?.format else {
                return noErr
            }
            
            let ablPointer = UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: audioBufferList))
            var remainingFrames = frameCount
            var currentOffset: AVAudioFrameCount = 0
            
            // 填充静音默认值
            for buffer in ablPointer {
                if let data = buffer.mData?.assumingMemoryBound(to: Float.self) {
                    memset(data, 0, Int(frameCount) * MemoryLayout<Float>.stride)
                }
            }
            
            // 从队列取帧并拆分输出
            while remainingFrames > 0, let buffer = self.frameQueue?.dequeue() {
                let framesToCopy = min(remainingFrames, buffer.frameLength)
                
                for bufferIndex in 0..<min(ablPointer.count, Int(processingFormat.channelCount)) {
                    let outBuffer = ablPointer[bufferIndex]
                    if let outData = outBuffer.mData?.assumingMemoryBound(to: Float.self),
                       let pcmData = buffer.floatChannelData?[bufferIndex] {
                        memcpy(outData + Int(currentOffset), pcmData, Int(framesToCopy) * MemoryLayout<Float>.stride)
                    }
                }
                
                remainingFrames -= framesToCopy
                currentOffset += framesToCopy
            }
            
            return noErr
        }
        
        audioEngine.attach(sinkNode)
        audioEngine.attach(sourceNode)
        
        audioEngine.connect(audioEngine.inputNode, to: sinkNode, format: inputFormat)
        audioEngine.connect(sourceNode, to: audioEngine.outputNode, format: processingFormat)
        
        // 启动引擎
        do {
            try audioEngine.start()
            self.isRecording = true
        } catch {
            self.errorMessage = "Failed to start audio engine: \(error.localizedDescription)"
            print(errorMessage ?? "")
        }
    }
    
    // 自定义音频处理函数：替换为你的业务逻辑
    private func processAudioBuffer(_ buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer {
        let processedBuffer = AVAudioPCMBuffer(pcmFormat: buffer.format, frameCapacity: buffer.frameLength)!
        processedBuffer.frameLength = buffer.frameLength
        
        // 示例：增益调整，替换为你的处理逻辑（如ONNX降噪）
        for channel in 0..<Int(buffer.format.channelCount) {
            guard let inputData = buffer.floatChannelData?[channel],
                  let outputData = processedBuffer.floatChannelData?[channel] else { continue }
            
            for frame in 0..<Int(buffer.frameLength) {
                outputData[frame] = inputData[frame] * 1.0 // 可替换为noiseCanceller?.process(...)
            }
        }
        
        return processedBuffer
    }
    
    func stopRecording() {
        audioEngine.stop()
        isRecording = false
        do {
            try AVAudioSession.sharedInstance().setActive(false)
        } catch {
            errorMessage = "Failed to deactivate audio session: \(error.localizedDescription)"
            print(errorMessage ?? "")
        }
    }
}

关键优化点说明 #

• 帧累积机制：确保每次处理的都是严格1024帧的缓冲区，满足自定义函数的输入要求
• 内存拷贝优化：使用memcpy替代逐帧循环拷贝，提升处理效率
• 音频路径简化：SinkNode仅负责采集累积，SourceNode负责输出，避免路径冲突
• 静音填充：当队列无数据时自动填充静音，避免输出爆音

内容的提问来源于stack exchange，提问作者amitt1236