Curl Multi添加大量Easy Handle时出现随机"无法连接服务器"问题

我尝试用Curl Multi下载十万余张地图切片，但出现随机的"无法连接到服务器"错误。只有限制每次调用curl_multi_perform前添加的新Easy Handle数量不超过512时才能正常运行，但这似乎不是预期用法？

原始代码 #

#include <string>
#include <iostream>
#include <vector>
#include <mutex>
#include <curl/curl.h>

std::mutex total_size_mutex;

static size_t response_callback(void *data, size_t size, size_t nmemb, void *userdata)
{
  size_t real_size = size * nmemb;
  size_t *total_size = (size_t *)userdata;
  std::lock_guard<std::mutex> lock(total_size_mutex);
  *total_size += real_size;
  return real_size;
}

// C++ main function
int main(int argc, char* argv[]){
  std::string access_token(argv[1]);
  access_token = std::string("Authorization: Bearer ") + access_token;
  struct curl_slist *chunk = NULL;
  chunk = curl_slist_append(chunk, access_token.c_str());
  chunk = curl_slist_append(chunk, "Connection: close");
  
  int sets = 50;
  int cols = 3;
  int rows = 795;
  int still_running = 1; /* keep number of running handles */
  size_t total_size = 0;
  
  curl_global_init(CURL_GLOBAL_ALL);
  CURLMcode mc;
  CURLMsg *msg; /* for picking up messages with the transfer status */
  int msgs_left; /* how many messages are left */
  CURL **m_handles = new CURL*[sets * cols * rows];
  CURLM *multi_handle = curl_multi_init();
  int numfds = 0;
  for(int d = 0; d < sets; ++d){
    for(int m = 0; m < cols; ++m){
        for(int n = 0; n < rows; ++n){
          char path[256];
          snprintf(path, 256, argv[2], d, m, n);
          m_handles[d * cols * rows + m * rows + n] = curl_easy_init();
          if(!m_handles[d * cols * rows + m * rows + n]) std::cout << "curl_easy_init() failed for " << d << m << n << std::endl;
          curl_easy_setopt(m_handles[d * cols * rows + m * rows + n], CURLOPT_WRITEFUNCTION, response_callback);
          curl_easy_setopt(m_handles[d * cols * rows + m * rows + n], CURLOPT_WRITEDATA, (void *)&total_size);
          curl_easy_setopt(m_handles[d * cols * rows + m * rows + n], CURLOPT_URL, path);
          curl_easy_setopt(m_handles[d * cols * rows + m * rows + n], CURLOPT_HTTPHEADER, chunk);
          curl_easy_setopt(m_handles[d * cols * rows + m * rows + n], CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
          mc = curl_multi_add_handle(multi_handle, m_handles[d * cols * rows + m * rows + n]);
          if(mc != CURLM_OK) std::cout << "curl_multi_add_handle: " << curl_multi_strerror(mc) << std::endl;
          if(++numfds > 512){  // THIS IS NEEDED TO AVOID COULDN'T CONNECT TO SERVER !!!
            numfds = 0;
            mc = curl_multi_perform(multi_handle, &still_running);
          }
      }
    }
  }
  
  do {
    mc = curl_multi_perform(multi_handle, &still_running);
    if(mc != CURLM_OK){
      std::cout << "curl_multi_perform: " << curl_multi_strerror(mc) << std::endl;
      break;
    }

    mc = curl_multi_poll(multi_handle, NULL, 0, 100, &numfds);
    if(mc != CURLM_OK) std::cout << "curl_multi_perform: " << curl_multi_strerror(mc) << std::endl;

    std::cout << still_running << "\t" << total_size << "\t" << numfds << std::endl;

    if(mc != CURLM_OK) break;
  } while(still_running);


    while((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
      if(msg->msg == CURLMSG_DONE) {
        int d, m, n;
        for(d = 0; d < sets; ++d){
          for(m = 0; m < cols; ++m){
            for(n = 0; n < rows; ++n){
              if(msg->easy_handle == m_handles[d * cols * rows + m * rows + n]) break;
            }
            if(msg->easy_handle == m_handles[d * cols * rows + m * rows + n]) break;
          }
          if(msg->easy_handle == m_handles[d * cols * rows + m * rows + n]) break;
        }

        if(msg->data.result){ 
          std::cout << d << "\t" << m << "\t" << n << "\t" << curl_easy_strerror(msg->data.result) << std::endl;
        }

      }
    }
  

  
  // Cleanup
  if(chunk){
    curl_slist_free_all(chunk);
  }
  for(int d = 0; d < sets; ++d){
    for(int m = 0; m < cols; ++m){
      for(int n = 0; n < rows; ++n){
        curl_multi_remove_handle(multi_handle, m_handles[d * cols * rows + m * rows + n]);
        curl_easy_cleanup(m_handles[d * cols * rows + m * rows + n]);
      }
    }
  }
  curl_multi_cleanup(multi_handle);
  curl_global_cleanup();
  delete[] m_handles;

  std::cout << "Total size downloaded: " << total_size << std::endl;
  
 
  return 0;
}  

问题触发点 #

移除以下代码段后，会出现大量"无法连接到服务器"错误：

if(++numfds > 512){  // THIS IS NEEDED TO AVOID COULDN'T CONNECT TO SERVER !!!
  numfds = 0;
  mc = curl_multi_perform(multi_handle, &still_running);
}

原因分析 #

一次性添加十万余Easy Handle会瞬间触发大量连接请求，导致两种核心问题：

• 系统资源耗尽：每个连接需要占用套接字（文件描述符），瞬间创建数千个套接字可能超出系统临时资源分配能力，导致连接初始化失败。
• 服务器端限流：目标服务器通常会限制短时间内来自同一IP的连接数，大量并发请求会被判定为恶意流量或超出服务配额，直接拒绝连接。

你添加的512限制有效，本质是分批启动连接，让Curl有时间处理已有的连接请求，避免瞬间压垮系统或触发服务器限流。

规范解决方案 #

1. 动态控制并发连接数（推荐）

不要一次性添加所有Handle，而是维护待下载队列，每次只添加固定数量的Handle到Multi中，当某个连接完成后再从队列补充新的Handle。示例调整逻辑：

// 伪代码核心逻辑
std::queue<CURL*> pending_handles;
// 预先创建所有Easy Handle并存入队列
for(...) {
  CURL* handle = curl_easy_init();
  // 设置handle参数（URL、头部等）
  pending_handles.push(handle);
}

const int MAX_CONCURRENT = 512;
int active_handles = 0;
// 初始添加一批Handle
while(active_handles < MAX_CONCURRENT && !pending_handles.empty()) {
  curl_multi_add_handle(multi_handle, pending_handles.front());
  pending_handles.pop();
  active_handles++;
}

// 主处理循环
do {
  curl_multi_perform(multi_handle, &still_running);
  curl_multi_poll(multi_handle, NULL, 0, 100, &numfds);
  
  // 检查完成的连接，补充新任务
  while((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
    if(msg->msg == CURLMSG_DONE) {
      active_handles--;
      // 清理当前完成的Handle
      curl_multi_remove_handle(multi_handle, msg->easy_handle);
      curl_easy_cleanup(msg->easy_handle);
      
      // 从队列添加新的Handle
      if(!pending_handles.empty()) {
        curl_multi_add_handle(multi_handle, pending_handles.front());
        pending_handles.pop();
        active_handles++;
      }
    }
  }
} while(still_running || !pending_handles.empty());

2. 配置Curl Multi并发限制

通过Curl内置选项直接限制总并发连接数，无需手动分批：

// 设置Multi Handle的最大总连接数
curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, 512);
// 设置单个Easy Handle的最大复用连接数
curl_easy_setopt(handle, CURLOPT_MAXCONNECTS, 10);

3. 优化HTTP连接复用

• 移除Connection: close头部，让Curl复用TCP连接，减少连接创建开销
• 确保服务器支持HTTP/2（你已启用CURL_HTTP_VERSION_2_0），HTTP/2可在单个TCP连接上处理多个请求，大幅降低并发连接压力

4. 调整系统资源限制（可选）

临时调高系统文件描述符限制（仅在必要时使用）：

# 临时调整当前shell的文件描述符限制
ulimit -n 4096

内容的提问来源于stack exchange，提问作者andrvibo