#include "sami_core"
int num_channels = input_file->getNumChannel();
int sample_rate = input_file->getSampleRate();
int num_frames = input_file->getNumFrames();
auto in_samples = loadWholeAudioFile(file_src);
std::vector<uint8_t> modelBin = loadBinaryFromFile(modelPath);
// create handle
SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
"enable_stereo":true,
"enable_pre_delay":true,
"enable_pre_process":true,
"enable_after_process":true,
"after_process_param": {"dtd_threshold":0.2,
"st_ramp":0.2,
"dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
std::cerr << "create handler failed: " << ret;
exit(-1);
}
SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;
//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;
SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;
for(int c = 0; c < int(num_channels); ++c) {
in_audio_buffer[0].data[c] = new float[pre_define_block_size];
in_audio_buffer[0].data[c] = new float[pre_define_block_size];
out_audio_buffer.data[c] = new float[pre_define_block_size];
}
SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;
SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;
for(;hasAudioSamples();)
{
copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
in_audio_buffer[0].numberSamples = feed_size;
in_audio_buffer[1].numberSamples = feed_size;
out_audio_buffer.numberSamples = feed_size;
ret = SAMICoreProcess(handle, &in_block, &out_block);
if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
std::cerr << "process error: " << ret;
exit(-1);
}
if(out_audio_buffer.numberSamples > 0) {
// do something after process
doSomethingAfterProcess(out_block); //业务从out_block拷贝处理后的数据
}
}
SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer && flushProperty.data) {
SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
// do something after process
doSomethingAfterProcess(out_block); //业务从out_block拷贝处理后的数据
}
}
SAMICoreDestroyProperty(&flushProperty);
SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);
// release resources
for(int c = 0; c < int(num_channels); ++c) {
delete[] in_audio_buffer[0].data[c];
delete[] in_audio_buffer[1].data[c];
delete[] out_audio_buffer.data[c];
}
delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;
即读取整个模型文件到内存,实现方法自由发挥,例子中loadModelAsBinary
仅供参考。算法模型详见回声消除介绍小节。
传入模型内存地址、模型大小、采样率和 maxBlockSize,通过 SAMICoreCreateHandleByIdentify
创建 handle。
SAMICoreExecutorContextCreateParameter
参数介绍参数 | 类型 | 说明 |
---|---|---|
sampleRate | int | 入参,指音频的采样率 |
maxBlockSize | int | 入参,每次输入音频的最大的大小,算法需要根据此字段提前分配内存等,接下来每次送入process的大小不能超过该值 |
numChannel | int | 入参,音频的通道数 |
modelBuffer | const char* | 入参,模型的内容 |
modelLen | int | 入参,模型的内容的长度 |
bussinessInfo | const char* | 入参, 表示调用的业务方信息 |
numAudioBuffer | int | 入参, 回声消除v3为2 |
configInfo | const char* | 入参, 见下文configInfo |
configInfo
参数 | 类型 | 说明 |
---|---|---|
utility | string | 入参,固定设置为CommonAecUtility |
enable_stereo | bool | 入参,默认值:false; |
enable_pre_delay | bool | 入参,默认值:false; |
enable_pre_process | bool | 入参,默认值:false; true时候打开防爆音前处理 |
enable_after_process | bool | 入参,默认值:false; true时候打开单双讲检测后处理,在单讲场景提供更好的效果 |
after_process_param | json-map | 入参,默认值:见下文 单双讲检测后处理的配置参数 |
参数 | 含义 | 范围 |
---|---|---|
dtd_threshold | 控制单双讲检测的阈值 | 0<=x<=1,默认取0.2 |
st_ramp | 控制检测到单讲后gain的衰退速率 | 0<=x<=1,默认取0.2 |
dt_ramp | 控制检测到双讲后gain的增益速率 | 0<=x<=1,默认取0.2 |
举例:
需要根据实际情况打开关闭功能
SAMICoreHandle handle = nullptr;
SAMICoreExecutorContextCreateParameter createParameter;
memset(&createParameter, 0, sizeof(SAMICoreExecutorContextCreateParameter));
createParameter.sampleRate = sample_rate;
createParameter.maxBlockSize = pre_define_block_size;
createParameter.numChannel = num_channels;
createParameter.modelBuffer = reinterpret_cast<char*>(modelBin.data());
createParameter.modelLen = modelBin.size();
createParameter.bussinessInfo = "aec_v3_demo";
createParameter.numAudioBuffer = 2;
createParameter.configInfo = R"( {"utility":"CommonAecUtility",
"enable_stereo":true,
"enable_pre_delay":true,
"enable_pre_process":true,
"enable_after_process":true,
"after_process_param": {"dtd_threshold":0.2,
"st_ramp":0.2,
"dt_ramp":0.2} } )";
int ret = SAMICoreCreateHandleByIdentify(&handle, SAMICoreIdentify_EngineExecutor_CE_AEC, &createParameter);
if(ret != SAMI_OK) {
std::cerr << "create handler failed: " << ret;
exit(-1);
}
有几种情况会导致创建失败:
模型数据不正确,例如模型数据损坏或者大小不对。
Block size 数据不正确。
SAMICoreAudioBuffer,用于存放音频数据,它支持 Planar-Float 和 Interleaved-Float 类型数据。更多关于音频数据格式请参看名词解释 部分。SAMICoreBlock,用于存放需要处理的数据。
目前接口内部有ringBuffer,所以支持任意block_size, 当数据量不足的时候,不会有数据返回。
注意:输入SAMICoreAudioBuffer 需申请 2个,下标0存放mic音频数据,下标1存放ref音频数据
SAMICoreAudioBuffer* in_audio_buffer = new SAMICoreAudioBuffer[2];
//存放mic数据
in_audio_buffer[0].numberChannels = num_channels;
in_audio_buffer[0].numberSamples = pre_define_block_size;
in_audio_buffer[0].data = new float*[num_channels];
in_audio_buffer[0].isInterleave = 0;
//存放ref数据
in_audio_buffer[1].numberChannels = num_channels;
in_audio_buffer[1].numberSamples = pre_define_block_size;
in_audio_buffer[1].data = new float*[num_channels];
in_audio_buffer[1].isInterleave = 0;
SAMICoreAudioBuffer out_audio_buffer;
out_audio_buffer.numberChannels = num_channels;
out_audio_buffer.numberSamples = pre_define_block_size;
out_audio_buffer.data = new float*[num_channels];
out_audio_buffer.isInterleave = isInterleave;
for(int c = 0; c < int(num_channels); ++c) {
in_audio_buffer[0].data[c] = new float[pre_define_block_size];
in_audio_buffer[0].data[c] = new float[pre_define_block_size];
out_audio_buffer.data[c] = new float[pre_define_block_size];
}
SAMICoreBlock in_block;
memset(&in_block, 0, sizeof(SAMICoreBlock));
in_block.numberAudioData = 2;
in_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
in_block.audioData = in_audio_buffer;
SAMICoreBlock out_block;
memset(&out_block, 0, sizeof(SAMICoreBlock));
out_block.numberAudioData = 1;
out_block.dataType = SAMICoreDataType::SAMICoreDataType_AudioBuffer;
out_block.audioData = &out_audio_buffer;
将待处理的音频数据拷贝到 in_audio_buffer
中,经过 SAMICoreProcess
处理后,结果将拷贝至 output 中。示例中采用这种方法。
for(;hasAudioSamples();)
{
copySamplesToInputBuffer(mic_data, in_audio_buffer[0]); //拷贝数据或者修改数据指针in_audio_buffer的指向
copySamplesToInputBuffer(ref_data, in_audio_buffer[1]);
in_audio_buffer[0].numberSamples = feed_size;
in_audio_buffer[1].numberSamples = feed_size;
out_audio_buffer.numberSamples = feed_size;
ret = SAMICoreProcess(handle, &in_block, &out_block);
if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
std::cerr << "process error: " << ret;
exit(-1);
}
if(out_audio_buffer.numberSamples > 0) {
// do something after process
doSomethingAfterProcess(out_block); //业务从out_block拷贝处理后的数据
}
}
更新音频数据的指针,指向正确的内存即可,这样可以避免内存数据的拷贝。
for(;hasAudioSamples();)
{
updateInputBuffer(mic_data, in_audio_buffer[0]);
updateInputBuffer(ref_data, in_audio_buffer[1]);
updateOutputBuffer(out_audio_buffer);
int ret = SAMICoreProcess(handle, &in_block, &out_block);
if(ret != SAMI_OK && ret != SAMI_ENGINE_INPUT_NEED_MORE_DATA) {
std::cerr << "process error: " << ret;
exit(-1);
}
if(out_audio_buffer.numberSamples > 0) {
// do something after process
doSomethingAfterProcess(out_block); //业务从out_block拷贝处理后的数据
}
}
有几种情况导致处理失败:
无效的 handle。handle 创建失败了,但仍然拿错误的 handle 进行 process
SAMICoreAudioBuffer 和 SAMICoreBlock 设置错误。
需要注意的是:当 block size 较小的情况下,前几帧会由于数据量不足无法处理而返回错误码SAMI_ENGINE_INPUT_NEED_MORE_DATA,但out_audio_buffer.numberSamples不一定为0
由于算法存在缓存部分数据,因此当音频已经全部送入算法后,需要调用这个接口,将所有缓存的数据取出
SAMICoreProperty flushProperty;
memset(&flushProperty, 0, sizeof(SAMICoreProperty));
flushProperty.type = SAMICoreDataType_AudioBuffer;
SAMICoreGetPropertyById(handle, SAMICorePropertyID_Common_Flush, &flushProperty);
if(flushProperty.dataLen > 0 && flushProperty.type == SAMICoreDataType_AudioBuffer && flushProperty.data) {
SAMICoreAudioBuffer* bufferArray = (SAMICoreAudioBuffer*)flushProperty.data;
if(bufferArray[0].data && bufferArray[0].numberSamples > 0) {
// do something after process
doSomethingAfterProcess(out_block); //业务从out_block拷贝处理后的数据
}
}
SAMICoreDestroyProperty(&flushProperty);
当处理完成后,后续有新数据需要处理,而又不想重复创建handle时,可以调用reset接口,之后就可以当做一个新实例环境来用
SAMICoreProperty resetProperty;
memset(&resetProperty, 0, sizeof(SAMICoreProperty));
resetProperty.id = SAMICorePropertyID_Common_Reset;
resetProperty.type = SAMICoreDataType_Null;
SAMICoreSetProperty(handle, SAMICorePropertyID_Common_Reset, &resetProperty);
释放 handle
ret = SAMICoreDestroyHandle(handle);
此外,还要注意音频数据数据的内存释放(如果有)。例如:
for(int c = 0; c < int(num_channels); ++c) {
delete[] in_audio_buffer[0].data[c];
delete[] in_audio_buffer[1].data[c];
delete[] out_audio_buffer.data[c];
}
delete[] in_audio_buffer.data[0];
delete[] in_audio_buffer.data[1];
delete[] in_audio_buffer;
delete[] out_audio_buffer.data;
SAMICoreDestroyHandle(handle);
handle = nullptr;