Intel MPI环境下Scatterv数据散射异常排查求助

我正在开发一款MPI应用，通过subarrays和Scatterv将浮点型全局域划分为多个tile并分发至不同rank。本地使用g++/icpc编译、OpenMPI启动时程序运行正常，但在集群环境（icpc编译、Intel MPI启动）中，尽管已验证counts和displacements参数正确，数据分发仍出现异常。

• 本地因核心数不足启动时需添加--oversubscribe参数，集群环境无需该参数，这是已知唯一差异，且理论上不应影响结果
• 问题仅在number_of_procs*4==domain_edge_size场景下触发，其他分解方式和tile尺寸均正常
• 已将程序简化为仅包含类型创建、子数组定义、散射操作的最小复现示例（无需同步），问题仍存在，推测可能与集群MPI配置相关

可复现代码 #

main.cpp

#include "mpi.h"
#include <vector>
#include <iostream>
#include <cmath>
#include <array>
#include <memory>

void run(std::vector<float> &out_result)  {
    int rank, size;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    int global_edge_size = 32;

    // fill the initial temperatures with some values
    std::vector<float> initial_temperature;
    for (int i = 0; i < global_edge_size * global_edge_size; i++) {
        initial_temperature.push_back(float(i));
    }

    // 1d decomposition
    // num_ranks in x and y direction
    std::array<int, 2> dims = {size, 1};
    int n_dims = 1;
    int tile_size_x = global_edge_size / size;
    int tile_size_y = global_edge_size;
    // tile with halo border
    int tile_size_with_halo_x = tile_size_x + (2 * 2);
    int tile_size_with_halo_y = tile_size_y + (2 * 2);
    int halo_zone_size = 2;

    MPI_Datatype global_tile_type_float, local_tile_type_float;
    
    // only root needs the global tile type
    if (rank == 0) {
        std::array domain_dims = {global_edge_size, global_edge_size};
        std::array tile_dims = {tile_size_y, tile_size_x};
        std::array<int, 2> start_arr = {0, 0};

        // initial datatypes to derive the resized types from
        MPI_Datatype tile_org_type_float{MPI_DATATYPE_NULL};
        MPI_Type_create_subarray(2, domain_dims.data(), tile_dims.data(), start_arr.data(), MPI_ORDER_C, MPI_FLOAT, &tile_org_type_float);
        MPI_Type_create_resized(tile_org_type_float, 0, 1 * sizeof(float), &global_tile_type_float);
        MPI_Type_commit(&global_tile_type_float);

        MPI_Type_free(&tile_org_type_float);
    }

    std::array<int, 2> local_tile_with_halo_dims = {tile_size_with_halo_y, tile_size_with_halo_x};
    std::array<int, 2> local_tile_dims = {tile_size_y, tile_size_x};
    std::array<int, 2> start_arr = {halo_zone_size, halo_zone_size};

    MPI_Type_create_subarray(2, local_tile_with_halo_dims.data(), local_tile_dims.data(), start_arr.data(), MPI_ORDER_C, MPI_FLOAT, &local_tile_type_float);
    MPI_Type_commit(&local_tile_type_float);

    // array of ones, each rank gets one tile
    auto counts = std::make_unique<int[]>(size);
    std::fill_n(counts.get(), size, 1);

    auto displacements = std::make_unique<int[]>(size);

    for (int i = 0; i < size; i++ ) {
        int row = i / dims[0];
        int col = i % dims[0];
        displacements[i] = (row * tile_size_y * global_edge_size) + (col * tile_size_x);
    }

    if (rank == 0) {
        printf("displacements: ");
        for (int i = 0; i < size; i++) {
            printf("%d, ", displacements[i]);
        }
        printf("\n");
    }

    std::vector<float> tile_temps(tile_size_with_halo_x * tile_size_with_halo_y);

    MPI_Scatterv(initial_temperature.data(), counts.get(), displacements.get(), global_tile_type_float, tile_temps.data(), 1, local_tile_type_float, 0, MPI_COMM_WORLD);
    
    // synchronization only for printing purposes
    MPI_Barrier(MPI_COMM_WORLD);
    if (rank == 0) {
        printf("[%d] printing %dx%d grid (initial temperature)\n", rank, global_edge_size, global_edge_size);
        for (int r = 0; r < global_edge_size; r++) {
            for (int c = 0; c < global_edge_size; c++) {
                printf("%f ", initial_temperature[r * global_edge_size + c]);
            }
            printf("\n");
        }
    }
    MPI_Barrier(MPI_COMM_WORLD);
    
    // print received data from each rank
    for (int i = 0; i < size; i++) {
        MPI_Barrier(MPI_COMM_WORLD);
        if (rank == i) {
            printf("[%d] printing %dx%d grid\n", rank, tile_size_with_halo_x, tile_size_with_halo_y);
            for (int r = halo_zone_size; r < tile_size_y + halo_zone_size; r++) {
                for (int c = halo_zone_size; c < tile_size_x + halo_zone_size; c++) {
                    printf("%f ", tile_temps[r * tile_size_with_halo_x + c]);
                }
                printf("\n");
            }
        }
    }
}

int main(int argc, char** argv) {
    MPI_Init(&argc, &argv);

    int size, rank;
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    std::vector<float> result;
    run(result);

    MPI_Finalize();
    return 0;
}

CMakeLists.txt

cmake_minimum_required(VERSION 3.20)

project(test LANGUAGES CXX)

set(CMAKE_CXX_STANDARD          17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

set(MPI_CXX_SKIP_MPICXX ON)

find_package(MPI REQUIRED COMPONENTS CXX)

add_executable(main ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
target_link_libraries(main MPI::MPI_CXX)

编译运行指令

cmake -Bbuild -S.
cmake --build build --config Release
mpirun -np 8 ./main

调试输出显示各进程接收的数据与预期不符，问题并非出在MPI_Gatherv环节。可提供更多代码或集群环境信息，恳请协助排查。

内容的提问来源于stack exchange，提问作者Pavel Kratochvil