C++ OpenMP+CPLEX并行程序线程增多性能下降求助

我开发了一个C++程序，核心函数包含一个for循环，每次迭代需求解一个CPLEX规划问题——这些问题既有共享约束，也有各自的专属约束。我使用#pragma omp parallel for实现并行求解，同时设置每个CPLEX环境仅使用1个线程；为避免伪共享，我为每个线程创建了所需变量的本地副本，并通过#pragma omp critical控制读写操作。

但实际运行中，单线程时性能最佳，线程数量增加后程序反而变慢。我尝试优化变量本地拷贝逻辑后，耗时略有改善，但线程增多时性能仍呈下降趋势。

void solving(vector<struct block> &blocks,
             vector<vector<double>> A,
             vector<double> b,
             vector<vector<double>> C,
             vector<double> V,
             vector<vector<double>> ZZ,
             vector<double> Mm,
             double l,
             bool &change,
             vector<bool> &sign) {

    int tt = th;

    omp_set_num_threads(tt);

#pragma omp parallel for
    for (int ii = 0; ii < tt; ii++) {

        int TT;

        vector<struct block> blocks1;
        vector<vector<double>> A1;
        vector<double> b1;
        vector<vector<double>> C1;
        vector<double> V1;
        vector<vector<double>> ZZ1;
        vector<double> Mm1;
        double l1;
        vector<bool> sign1;
        double nvar1;
        int p1;
        int m1;

#pragma omp critical
        {
            TT = th;
            blocks1 = blocks;
            A1 = A;
            b1 = b;
            C1 = C;
            V1 = V;
            ZZ1 = ZZ;
            Mm1 = Mm;
            l1 = l;
            sign1 = sign;
            nvar1 = nvar;
            p1 = p;
            m1 = m;
        }

        IloEnv myenv1;            // environment object
        IloModel mymodel(myenv1); // model object

        IloNumVarArray X(myenv1, nvar1);
        for (int i = 0; i < nvar1; i++) {
            // X[i] = IloNumVar(myenv1, 0, IloInfinity, ILOINT);
            X[i] = IloNumVar(myenv1, 0, 1, ILOBOOL);
        }

        IloArray<IloNumVarArray> Y(myenv1, l1);
        for (int i = 0; i < l1; i++) {
            Y[i] = IloNumVarArray(myenv1, p1);
            for (int j = 0; j < p; j++) {
                Y[i][j] = IloNumVar(myenv1, 0, 1, ILOBOOL);
            }
        }

        /* constraints diffrent in each problem*/

        IloExprArray exp1(myenv1, m1);

        for (int i = 0; i < m1; i++) {
            exp1[i] = IloExpr(myenv1);
        }

        for (int i = 0; i < m1; i++) {
            for (int k = 0; k < nvar1; k++) {
                exp1[i] += A1[i][k] * X[k];
            }

            mymodel.add(exp1[i] <= b1[i]);
        }

        // Y

        /* constraints diffrent in each problem*/

        IloExprArray exp3(myenv1, l1 * (p1 + 1.0));

        for (int i = 0; i < l1 * (p1 + 1.0); i++) {
            exp3[i] = IloExpr(myenv1);
        }

        for (int i = 0; i < l1; i++) {
            for (int j = 0; j < p1; j++) {
                int k = j + (i * p1);
                exp3[k] = (ZZ1[i][j] + 1) * Y[i][j] + (1 - Y[i][j]) * (-M);
            }
        }

        IloExprArray exp4(myenv1, p1 * l1);

        for (int i = 0; i < p1 * l1; i++) {
            exp4[i] = IloExpr(myenv1);
        }

        for (int k = 0; k < l1; k++) {

            for (int i = 0; i < p1; i++) {
                for (int j = 0; j < nvar1; j++) {
                    int s = i + (k * p1);
                    exp4[s] += C1[i][j] * X[j];
                }
            }
        }

        for (int i = 0; i < l1 * p1; i++) {
            mymodel.add(exp4[i] >= exp3[i]);
        }

        // Objedctive function
        IloExpr exp(myenv1);
        for (int i = 0; i < nvar1; i++) {
            exp += V1[i] * X[i];
        }

        mymodel.add(IloMaximize(myenv1, exp));

        // solve
        IloCplex mycplex(myenv1);
        mycplex.extract(mymodel);
        mycplex.setParam(IloCplex::Param::Threads, 1); //////////////////////
        // to delete the writing
        mycplex.setOut(myenv1.getNullStream());

        IloBool feasible =
            mycplex.solve(); // solves model and stores
                             // whether or not it is feasible in an IloBool
                             // variable called "feasible"

        if (feasible == IloTrue) {

            for (int i = 0; i < p1; i++) {
                blocks1[ii].Z[i] = 0;
            }
            // value of x
            for (int i = 0; i < nvar1; i++) {
                blocks1[ii].X[i] = round(mycplex.getValue(X[i]));
            }

        } else {

            blocks1[ii].test1 = true;
            for (int i = 0; i < nvar1; i++) {
                blocks1[ii].X[i] = -M;
            }
            for (int i = 0; i < p1; i++) {

                blocks1[ii].Z[i] = -M;
            }
        }

        // Closing the Model
        mycplex.clear();
        myenv1.end();

#pragma omp critical
        {

            blocks[ii].test1 = blocks1[ii].test1;
            for (int i = 0; i < nvar; i++) {
                blocks[ii].X[i] = blocks1[ii].X[i];
            }

            for (int i = 0; i < p; i++) {
                blocks[ii].Z[i] = blocks1[ii].Z[i];
            }
        }
    }
}

• 临界区成为性能瓶颈：第一个#pragma omp critical中执行了大量容器的完整拷贝（如blocks1 = blocks、A1 = A等），所有线程必须排队等待进入该区域，完全抵消了并行计算的优势。实际上，这些共享数据无需每个线程拷贝，直接通过引用访问即可，仅需为每个线程的专属数据创建本地副本。
• CPLEX环境重复初始化开销过大：每个线程都创建并销毁IloEnv、IloModel、IloCplex对象，这部分初始化/销毁的开销非常高。可以使用**线程局部存储（TLS）**或线程池复用这些对象，避免重复创建。
• 循环迭代逻辑不合理：当前循环次数绑定线程数（ii < tt），正确的做法应该是根据待求解问题的数量设置循环次数，让OpenMP自动分配线程处理任务，而非强制每个线程仅处理一个问题。
• 不必要的容器拷贝：blocks1是整个blocks容器的拷贝，但每个线程仅需修改blocks[ii]对应的单个元素，完全无需拷贝整个容器，只需在本地创建单个block对象，求解完成后直接写回blocks[ii]即可。
• 内存与缓存效率低下：大量容器拷贝会导致内存占用激增，触发频繁的页交换，同时破坏CPU缓存的局部性，这也是线程越多性能越差的重要原因。

void solving(vector<struct block> &blocks,
             const vector<vector<double>>& A,  // 改为const引用避免拷贝
             const vector<double>& b,
             const vector<vector<double>>& C,
             const vector<double>& V,
             const vector<vector<double>>& ZZ,
             const vector<double>& Mm,
             double l,
             bool &change,
             const vector<bool>& sign) {

    int problem_count = blocks.size();  // 待求解问题数量为blocks的大小
    int tt = th;
    omp_set_num_threads(tt);

    // 线程局部存储CPLEX环境，复用对象
    #pragma omp parallel
    {
        IloEnv myenv1;
        IloCplex mycplex(myenv1);
        mycplex.setParam(IloCplex::Param::Threads, 1);
        mycplex.setOut(myenv1.getNullStream());

        #pragma omp for
        for (int ii = 0; ii < problem_count; ii++) {  // 遍历所有问题，而非线程数
            struct block local_block = blocks[ii];  // 仅拷贝单个元素，而非整个容器
            double nvar1 = nvar;
            int p1 = p;
            int m1 = m;
            double l1 = l;

            IloModel mymodel(myenv1);

            // 变量定义、约束添加、目标函数构建逻辑（与原代码一致，但直接使用共享数据的引用）
            IloNumVarArray X(myenv1, nvar1);
            for (int i = 0; i < nvar1; i++) {
                X[i] = IloNumVar(myenv1, 0, 1, ILOBOOL);
            }

            IloArray<IloNumVarArray> Y(myenv1, l1);
            for (int i = 0; i < l1; i++) {
                Y[i] = IloNumVarArray(myenv1, p1);
                for (int j = 0; j < p1; j++) {
                    Y[i][j] = IloNumVar(myenv1, 0, 1, ILOBOOL);
                }
            }

            IloExprArray exp1(myenv1, m1);
            for (int i = 0; i < m1; i++) {
                exp1[i] = IloExpr(myenv1);
                for (int k = 0; k < nvar1; k++) {
                    exp1[i] += A[i][k] * X[k];  // 直接用A的引用，无需拷贝
                }
                mymodel.add(exp1[i] <= b[i]);
            }

            // 其余约束、目标函数构建逻辑同理，直接使用共享数据引用

            mycplex.extract(mymodel);
            IloBool feasible = mycplex.solve();

            if (feasible == IloTrue) {
                for (int i = 0; i < p1; i++) {
                    local_block.Z[i] = 0;
                }
                for (int i = 0; i < nvar1; i++) {
                    local_block.X[i] = round(mycplex.getValue(X[i]));
                }
            } else {
                local_block.test1 = true;
                for (int i = 0; i < nvar1; i++) {
                    local_block.X[i] = -M;
                }
                for (int i = 0; i < p1; i++) {
                    local_block.Z[i] = -M;
                }
            }

            // 仅写回单个元素，无需临界区（因为每个线程操作的blocks[ii]是唯一的）
            blocks[ii] = local_block;

            // 清理当前模型，复用环境和求解器
            mymodel.removeAll();
            X.end();
            Y.end();
            exp1.end();
            // 其余表达式对象也需要正确清理
        }

        mycplex.clear();
        myenv1.end();
    }
}

内容的提问来源于stack exchange，提问作者Bederina Med