You need to enable JavaScript to run this app.
导航
【C】回声消除-V3
最近更新时间:2023.07.07 11:16:16首次发布时间:2023.03.16 11:40:12
完整例子
#include "sami_core"

int num_channels = input_file->getNumChannel();
int sample_rate = input_file->getSampleRate();
int num_frames = input_file->getNumFrames();
auto in_samples = loadWholeAudioFile(file_src);
std::vector<uint8_t> modelBin = loadBinaryFromFile(modelPath);

// create handle
SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
                                    "enable_stereo":true,
                                    "enable_pre_delay":true,
                                    "enable_pre_process":true,
                                    "enable_after_process":true,
                                    "after_process_param": {"dtd_threshold":0.2, 
                                                            "st_ramp":0.2,
                                                            "dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
    std::cerr << "create handler failed: " << ret;
    exit(-1);
}

SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;

//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;

SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;

for(int c = 0; c < int(num_channels); ++c) {
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    out_audio_buffer.data[c] = new float[pre_define_block_size];
}

SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;

SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;

for(;hasAudioSamples();)
{
    copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
    copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
    in_audio_buffer[0].numberSamples = feed_size;
    in_audio_buffer[1].numberSamples = feed_size;
    out_audio_buffer.numberSamples = feed_size;
    ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer  && flushProperty.data) {
    SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
    if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}
SAMICoreDestroyProperty(&flushProperty);

SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);

// release resources
for(int c = 0; c < int(num_channels); ++c) {
    delete[] in_audio_buffer[0].data[c];
    delete[] in_audio_buffer[1].data[c];
    delete[] out_audio_buffer.data[c];
}

delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;
使用步骤

〇、从文件中读取模型

即读取整个模型文件到内存,实现方法自由发挥,例子中loadModelAsBinary仅供参考。算法模型详见回声消除介绍小节。

一、创建算法句柄

传入模型内存地址、模型大小、采样率和 maxBlockSize,通过 SAMICoreCreateHandleByIdentify 创建 handle。

  • SAMICoreExecutorContextCreateParameter参数介绍
参数类型说明
sampleRateint入参,指音频的采样率
maxBlockSizeint入参,每次输入音频的最大的大小,算法需要根据此字段提前分配内存等,接下来每次送入process的大小不能超过该值
numChannelint入参,音频的通道数
modelBufferconst char*入参,模型的内容
modelLenint入参,模型的内容的长度
bussinessInfoconst char*入参, 表示调用的业务方信息
numAudioBufferint入参, 回声消除v3为2
configInfoconst char*入参, 见下文configInfo

configInfo

参数类型说明
utilitystring入参,固定设置为CommonAecUtility

enable_stereo

bool

入参,默认值:false;
当处理两个通道的数据时候,enable_stereo为ture,两个通道单独处理;enable_stereo为false,处理第一个通道后,拷贝结果覆盖第二个通道,节省一半计算量;

enable_pre_delay

bool

入参,默认值:false;
算法需要送入足够的数据才会输出结果,在实时场景需要等进等出,enable_pre_delay=true,会在一开始返回静音缓冲数据,减少接入难度,建议rtc场景默认开启

enable_pre_processbool入参,默认值:false; true时候打开防爆音前处理
enable_after_processbool入参,默认值:false; true时候打开单双讲检测后处理,在单讲场景提供更好的效果
after_process_paramjson-map入参,默认值:见下文 单双讲检测后处理的配置参数
  • after_process_param参数介绍
参数含义范围
dtd_threshold控制单双讲检测的阈值0<=x<=1,默认取0.2
st_ramp控制检测到单讲后gain的衰退速率0<=x<=1,默认取0.2
dt_ramp控制检测到双讲后gain的增益速率0<=x<=1,默认取0.2

举例:
需要根据实际情况打开关闭功能

SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
                                    "enable_stereo":true,
                                    "enable_pre_delay":true,
                                    "enable_pre_process":true,
                                    "enable_after_process":true,
                                    "after_process_param": {"dtd_threshold":0.2, 
                                                            "st_ramp":0.2,
                                                            "dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
    std::cerr << "create handler failed: " << ret;
    exit(-1);
}

有几种情况会导致创建失败:

  1. 模型数据不正确,例如模型数据损坏或者大小不对。

  2. Block size 数据不正确。

二、创建 SAMICoreBlock 用于存放输入和输出

SAMICoreAudioBuffer,用于存放音频数据,它支持 Planar-FloatInterleaved-Float 类型数据。更多关于音频数据格式请参看名词解释 部分。SAMICoreBlock,用于存放需要处理的数据。
目前接口内部有ringBuffer,所以支持任意block_size, 当数据量不足的时候,不会有数据返回。
注意:输入SAMICoreAudioBuffer 需申请 2个,下标0存放mic音频数据,下标1存放ref音频数据

SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;

//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;

SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;

for(int c = 0; c < int(num_channels); ++c) {
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    in_audio_buffer[0].data[c] = new float[pre_define_block_size];
    out_audio_buffer.data[c] = new float[pre_define_block_size];
}

SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;

SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;

三、处理音频

拷贝数据进行处理

将待处理的音频数据拷贝到 in_audio_buffer 中,经过 SAMICoreProcess 处理后,结果将拷贝至 output 中。示例中采用这种方法。

for(;hasAudioSamples();)
{
    copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
    copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
    in_audio_buffer[0].numberSamples = feed_size;
    in_audio_buffer[1].numberSamples = feed_size;
    out_audio_buffer.numberSamples = feed_size;
    ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

In-Place 进行处理

更新音频数据的指针,指向正确的内存即可,这样可以避免内存数据的拷贝。

for(;hasAudioSamples();)
{
    updateInputBuffer(mic_data, in_audio_buffer[0]);
    updateInputBuffer(ref_data, in_audio_buffer[1]);
    updateOutputBuffer(out_audio_buffer);
    
    int ret = SAMICoreProcess(handle, &in_block, &out_block);
    if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
        std::cerr << "process error: " << ret;
        exit(-1);
    }
    if(out_audio_buffer.numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}

有几种情况导致处理失败:

  1. 无效的 handle。handle 创建失败了,但仍然拿错误的 handle 进行 process

  2. SAMICoreAudioBuffer 和 SAMICoreBlock 设置错误。

需要注意的是:当 block size 较小的情况下,前几帧会由于数据量不足无法处理而返回错误码SAMI_ENGINE_INPUT_NEED_MORE_DATA,但out_audio_buffer.numberSamples不一定为0

四、获取最后结果

由于算法存在缓存部分数据,因此当音频已经全部送入算法后,需要调用这个接口,将所有缓存的数据取出

SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer  && flushProperty.data) {
    SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
    if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
        // do something after process
        doSomethingAfterProcess(out_block);  //业务从out_block拷贝处理后的数据
    }
}
SAMICoreDestroyProperty(&flushProperty);

五、重置算法句柄

当处理完成后,后续有新数据需要处理,而又不想重复创建handle时,可以调用reset接口,之后就可以当做一个新实例环境来用

SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);

六、释放资源

释放 handle

ret = SAMICoreDestroyHandle(handle);

此外,还要注意音频数据数据的内存释放(如果有)。例如:

for(int c = 0; c < int(num_channels); ++c) {
    delete[] in_audio_buffer[0].data[c];
    delete[] in_audio_buffer[1].data[c];
    delete[] out_audio_buffer.data[c];
}

delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;