MPI发送自定义类的std::vector时的未定义行为及内存问题求助

使用MPI_Type_create_struct注册了包含两个成员的自定义类Dummy，当std::vector<Dummy>的大小增长到20000时（该值随系统可用栈空间变化），程序抛出如下错误：

Read -1, expected 800000, errno = 14
Read -1, expected 800000, errno = 14
*** Process received signal ***
Signal: Segmentation fault (11)
Signal code: Address not mapped (1)

较小的vector大小运行正常；修改类定义添加更大成员后，单进程创建200万元素的std::vector<Dummy>会抛出std::bad_alloc异常。推测问题出在直接用std::vector<Dummy>作为发送缓冲区，但MPI无法使用指针方案，寻求解决方案。

#include <mpi.h> 
#include <vector>
#include <stddef.h> 
#include <assert.h>
#include <math.h>
using namespace std; 
typedef int64_t emInt;
class Dummy
{
private:
    emInt N ; 
    emInt Array[2][2]; 
public:
    
    Dummy(const int64_t nDivs) : 
    N(nDivs)
    {

    };
    friend MPI_Datatype register_mpi_type(Dummy const&); 
    ~Dummy() {}; 
};

int main (int argc, char* argv[])
{
    
    MPI_Init(&argc, &argv);     
    int rank , size ;
    int tag=0 ; 
    int flag=0 ;   
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Request request;
    MPI_Status status;
    Dummy object (1); 
    MPI_Datatype dummyType = register_mpi_type(object);

    std::size_t containerSize = 20000;  

    std::vector<int> neighbrs = {0, 1, 2, 3};
    for(auto itri:neighbrs)
    {
        int target = itri; 
        if(rank!= target)
        {
            std::vector<Dummy> TobeSend  (containerSize, Dummy(target)); 
            MPI_Isend(TobeSend.data(),containerSize,dummyType,
          target, tag, MPI_COMM_WORLD, &request ) ; 
            
        }

    }
    for(auto isource:neighbrs)
    {
        int source = isource;
        if(rank!= source)
        {
            std::vector<Dummy> TobeRecvd (containerSize ,Dummy(0));
            MPI_Irecv(TobeRecvd.data(),containerSize, dummyType ,source,tag,
            MPI_COMM_WORLD, &request); 
        }
    }
    MPI_Finalize(); 
    return 0; 
}

MPI_Datatype register_mpi_type(Dummy const&)
{
    Dummy object(1);

    MPI_Datatype builtType;

    constexpr std::size_t numMembers = 2;

    MPI_Datatype types[numMembers]=
    {
        MPI_INT64_T,
        MPI_INT64_T
    }
    ;

    int arrayOfBlockLengths[numMembers];

    arrayOfBlockLengths[0] = 1; 
    arrayOfBlockLengths[1] = 4; 

    MPI_Aint baseadress ; 

    MPI_Aint arrayOfDisplacements [numMembers] 
    = 
    {
        offsetof (Dummy,N),
        offsetof (Dummy,Array),
    };
    MPI_Type_create_struct(numMembers,arrayOfBlockLengths,arrayOfDisplacements,types,&builtType);
    MPI_Type_commit(&builtType);
    return builtType; 
};

#0  0x00007ffff7b6e94d in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007ffff4d0b244 in ?? () from /usr/lib/x86_64-linux-gnu/openmpi/lib/openmpi3/mca_btl_vader.so
#2  0x00007ffff4769556 in mca_pml_ob1_send_request_schedule_once () from /usr/lib/x86_64-linux-gnu/openmpi/lib/openmpi3/mca_pml_ob1.so
#3  0x00007ffff4767811 in mca_pml_ob1_recv_frag_callback_ack () from /usr/lib/x86_64-linux-gnu/openmpi/lib/openmpi3/mca_pml_ob1.so
#4  0x00007ffff4d0fae5 in mca_btl_vader_poll_handle_frag () from /usr/lib/x86_64-linux-gnu/openmpi/lib/openmpi3/mca_btl_vader.so
#5  0x00007ffff4d0fdb1 in ?? () from /usr/lib/x86_64-linux-gnu/openmpi/lib/openmpi3/mca_btl_vader.so
#6  0x00007ffff7942714 in opal_progress () from /lib/x86_64-linux-gnu/libopen-pal.so.40
#7  0x00007ffff7e9fc0d in ompi_mpi_finalize () from /lib/x86_64-linux-gnu/libmpi.so.40
#8  0x000055555555dd9c in main (argc=1, argv=0x7fffffffd438) at classicDataType.cpp:159

核心问题分析 #

1. 异步通信的缓冲区生命周期不匹配：循环内创建的TobeSend和TobeRecvd是局部变量，循环结束后立即被销毁，但MPI_Isend/MPI_Irecv是异步操作，MPI底层仍在访问已释放的内存，直接导致段错误。
2. 超大内存分配失败：一次性分配200万个Dummy对象的连续内存块，容易触发堆内存不足，抛出std::bad_alloc。

具体修复步骤 #

1. 延长缓冲区生命周期至通信完成 #

将缓冲区和请求对象移到循环外存储，等待所有异步操作完成后再释放：

int main (int argc, char* argv[])
{
    MPI_Init(&argc, &argv);     
    int rank , size ;
    int tag=0 ;   
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Datatype dummyType = register_mpi_type(Dummy(1));

    std::size_t containerSize = 20000;  
    std::vector<int> neighbrs = {0, 1, 2, 3};

    // 存储所有发送缓冲区和请求
    std::vector<std::vector<Dummy>> sendBuffers;
    std::vector<MPI_Request> sendReqs;
    // 存储所有接收缓冲区和请求
    std::vector<std::vector<Dummy>> recvBuffers;
    std::vector<MPI_Request> recvReqs;

    for(auto itri:neighbrs)
    {
        int target = itri; 
        if(rank!= target)
        {
            sendBuffers.emplace_back(containerSize, Dummy(target));
            MPI_Request req;
            MPI_Isend(sendBuffers.back().data(), containerSize, dummyType,
                      target, tag, MPI_COMM_WORLD, &req);
            sendReqs.push_back(req);
        }
    }

    for(auto isource:neighbrs)
    {
        int source = isource;
        if(rank!= source)
        {
            recvBuffers.emplace_back(containerSize, Dummy(0));
            MPI_Request req;
            MPI_Irecv(recvBuffers.back().data(), containerSize, dummyType,
                      source, tag, MPI_COMM_WORLD, &req);
            recvReqs.push_back(req);
        }
    }

    // 等待所有异步操作完成
    if(!sendReqs.empty())
        MPI_Waitall(sendReqs.size(), sendReqs.data(), MPI_STATUSES_IGNORE);
    if(!recvReqs.empty())
        MPI_Waitall(recvReqs.size(), recvReqs.data(), MPI_STATUSES_IGNORE);

    MPI_Type_free(&dummyType); // 记得释放自定义MPI类型
    MPI_Finalize(); 
    return 0; 
}

2. 优化内存分配避免std::bad_alloc #

• 分批次通信：如果200万元素必须传输，拆分成多个小批次发送/接收，减少单次内存分配压力。
• 预分配内存：用reserve提前为vector分配内存，减少内存碎片化：

sendBuffers.reserve(neighbrs.size());
for(auto itri:neighbrs)
{
    if(rank!= itri)
    {
        sendBuffers.emplace_back();
        sendBuffers.back().reserve(containerSize);
        for(std::size_t i=0; i<containerSize; ++i)
        {
            sendBuffers.back().emplace_back(itri);
        }
        // ... 发送逻辑
    }
}

3. 完善MPI类型注册（可选但更健壮） #

不需要创建临时Dummy对象，直接通过类型计算偏移量：

MPI_Datatype register_mpi_type()
{
    MPI_Datatype builtType;
    constexpr std::size_t numMembers = 2;
    MPI_Datatype types[numMembers] = {MPI_INT64_T, MPI_INT64_T};
    int arrayOfBlockLengths[numMembers] = {1, 4};
    MPI_Aint arrayOfDisplacements[numMembers] = {
        offsetof(Dummy, N),
        offsetof(Dummy, Array)
    };
    MPI_Type_create_struct(numMembers, arrayOfBlockLengths, arrayOfDisplacements, types, &builtType);
    MPI_Type_commit(&builtType);
    return builtType;
}

main中调用改为：MPI_Datatype dummyType = register_mpi_type();

4. 初始化类成员（可选） #

避免MPI读取未初始化的内存，完善构造函数：

Dummy(const int64_t nDivs) : N(nDivs)
{
    for(int i=0; i<2; ++i)
        for(int j=0; j<2; ++j)
            Array[i][j] = 0;
};

内容的提问来源于stack exchange，提问作者MA19