使用HDFql向HDF5复合数据集顺序写入可变长度字节数据的问题

问题说明 #

我尝试在HDF5中写入简单结构体数据，但始终无法向已创建的数据集新增记录，核心需求是存储任意长度的字节集合及其对应长度值。

原问题代码 #

void WriteStructToFile(vector<vector<char>> fl_stream)
{
    struct data
    {
        int size_array;
        char * value_array;
    };
    int status;
    string  query;
    string FileName = "Fly_stream_file.h5";
    string NameGroup = "Stream";
    string Namedataset = "Inclusions";

 //------------------------------- 创建文件和数据集 -------------
    query = "CREATE TRUNCATE FILE " + FileName;
    status = HDFql::execute(query.c_str());
    query = "USE FILE " + FileName;
    status = HDFql::execute(query.c_str());
    query = "CREATE GROUP " + NameGroup;
    status = HDFql::execute(query.c_str());
    query = "CREATE TRUNCATE CHUNKED DATASET "+ Namedataset +" AS COMPOUND(size_array AS INT, value_array AS VARCHAR)(UNLIMITED)";
    stringstream scriptst;
    scriptst <<"CREATE TRUNCATE CHUNKED DATASET "<< Namedataset <<" AS COMPOUND("<< "size_array AS INT OFFSET " << offsetof(struct data, size_array)<<" ,"<<" value_array AS VARCHAR OFFSET "<< offsetof(struct data, value_array) <<")(UNLIMITED)"<< " SIZE "<< sizeof(struct data);
    status = HDFql::execute(scriptst);
    // 清空请求
    scriptst.str(std::string());
    scriptst.clear();

//--------------- 填充数据集 ----------------
    // 此处模拟函数中数据到达的场景
    
    
    struct data realdata;
    int number = HDFql::variableRegister(&realdata);
    for (int i = 0;i < fl_stream.size();i++)
    {

        realdata.size_array = fl_stream[i].size();
        realdata.value_array = fl_stream[i].data();

        // 将数据写入数据集
        scriptst << "INSERT INTO " << Namedataset << "(-1) VALUES FROM MEMORY " << number<< " SIZE " << sizeof(struct data) << " OFFSET(" << offsetof(struct data, size_array) << ", "
<< offsetof(struct data,value_array) << ")";
        status = HDFql::execute(scriptst);

        HDFql::execute("ALTER DIMENSION Inclusions TO +1");
        scriptst.str(std::string());
        scriptst.clear();
    }
    HDFql::variableUnregister(&realdata);
status = HDFql::execute("CLOSE FILE");
}

问题分析与修复 #

代码存在三处核心错误：

1. 重复创建Inclusions数据集，且创建的数据集未放在提前创建的Stream分组下，导致分组冗余无用
2. INSERT语句用(-1)追加写入时HDFql会自动扩容数据集，额外执行ALTER DIMENSION会导致数据集多扩一条空记录
3. INSERT语句中不需要重复指定结构体的SIZE和OFFSET，创建数据集时已经完成了结构体与复合类型的映射，重复指定会导致字段读取错误

修正后代码 #

void WriteStructToFile(vector<vector<char>> fl_stream)
{
    struct data
    {
        int size_array;
        char * value_array;
    };
    int status;
    string  query;
    string FileName = "Fly_stream_file.h5";
    string NameGroup = "Stream";
    string Namedataset = NameGroup + "/Inclusions"; // 数据集放在Stream分组下

    //------------------------------- 创建文件和数据集 -------------
    query = "CREATE TRUNCATE FILE " + FileName;
    status = HDFql::execute(query.c_str());
    query = "USE FILE " + FileName;
    status = HDFql::execute(query.c_str());
    query = "CREATE GROUP " + NameGroup;
    status = HDFql::execute(query.c_str());
    
    stringstream scriptst;
    // 只创建一次数据集，指定结构体偏移和大小
    scriptst << "CREATE TRUNCATE CHUNKED DATASET " << Namedataset << " AS COMPOUND("
             << "size_array AS INT OFFSET " << offsetof(struct data, size_array) << ", "
             << "value_array AS VARCHAR OFFSET " << offsetof(struct data, value_array) 
             << ")(UNLIMITED) SIZE " << sizeof(struct data);
    status = HDFql::execute(scriptst);
    // 清空请求
    scriptst.str(std::string());
    scriptst.clear();

    //--------------- 填充数据集 ----------------
    struct data realdata;
    int number = HDFql::variableRegister(&realdata);
    for (int i = 0; i < fl_stream.size(); i++)
    {
        realdata.size_array = fl_stream[i].size();
        realdata.value_array = fl_stream[i].data();

        // 简化INSERT语句，无需额外指定SIZE和OFFSET
        scriptst << "INSERT INTO " << Namedataset << "(-1) VALUES FROM MEMORY " << number;
        status = HDFql::execute(scriptst);

        // 删掉多余的ALTER DIMENSION语句
        scriptst.str(std::string());
        scriptst.clear();
    }
    HDFql::variableUnregister(&realdata);
    status = HDFql::execute("CLOSE FILE");
}

修正后即可正常向复合数据集中追加任意长度的字节数据及其长度，数据会统一存储在Stream分组下的Inclusions数据集中。

内容的提问来源于stack exchange，提问作者OlegMart