如何使用curl无线程并行下载多个GitHub仓库文件

我需要用私有Token从GitHub仓库下载多个文件，已经实现了逐个下载的功能，但想在不使用线程的前提下实现多文件并行下载，可使用任意版本的Curl。当前的C++实现代码如下：

#include <curl/curl.h>

static size_t WriteMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
    size_t realsize = size * nmemb;
    auto& mem = *static_cast<std::string*>(userp);
    mem.append(static_cast<char*>(contents), realsize);
    SecureZeroMemory(contents, realsize);
    return realsize;
}

void curl(std::string& data, const std::string_view& url)
{
    CURL* curl_handle;
    CURLcode res;
    struct curl_slist* slist{};

    curl_handle = curl_easy_init();

    curl_easy_setopt(curl_handle, CURLOPT_URL, url.data());
    //curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPALIVE, 0);

    slist = curl_slist_append(slist, gitToken.data());
    slist = curl_slist_append(slist, "Accept: application/vnd.github.v3.raw");
    curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, slist);

    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &data);
    curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "curl/7.55.1");

    // Necessary for downloading file from github repo.
    curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
   
    curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L); // only to debug

    res = curl_easy_perform(curl_handle);
    OutputDebugStringA(curl_easy_strerror(res));

    if(res != CURLE_OK)
       std::cerr << "curl_easy_perform() failed: " << curl_easy_strerror(res) << '\n';

    curl_easy_cleanup(curl_handle);
    curl_global_cleanup();
}

int main()
{
    std::vector<std::string> urlList =
    {
        "...",
        "...",
        // ...
    };

    for (int i = 0; i < urlList.size(); i++)
    {
        std::string data;
        curl(data, urlList[i]);
        // save data to disk ...
    }
}

要实现无线程的并行下载，最适合的方案是使用Curl的Multi接口——它允许在单个线程内同时管理多个HTTP请求，底层通过事件驱动或轮询机制实现并行处理，不需要额外创建线程。

以下是修改后的完整实现代码：

#include <curl/curl.h>
#include <vector>
#include <string>
#include <iostream>
#include <windows.h> // 仅用于OutputDebugStringA，不需要可移除

static size_t WriteMemoryCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
    size_t realsize = size * nmemb;
    auto& mem = *static_cast<std::string*>(userp);
    mem.append(static_cast<char*>(contents), realsize);
    SecureZeroMemory(contents, realsize);
    return realsize;
}

// 封装每个下载任务的信息
struct DownloadTask {
    std::string url;
    std::string data;
    CURL* easy_handle = nullptr;
};

int main()
{
    // 全局Curl初始化，仅调用一次
    curl_global_init(CURL_GLOBAL_ALL);

    CURLM* multi_handle = curl_multi_init();
    if (!multi_handle) {
        std::cerr << "curl_multi_init() failed" << std::endl;
        curl_global_cleanup();
        return 1;
    }

    // 私有Token，替换为你的实际Token
    const std::string gitToken = "Authorization: token YOUR_PRIVATE_GITHUB_TOKEN";

    std::vector<DownloadTask> tasks = {
        {"https://api.github.com/repos/OWNER/REPO/contents/file1.txt", {}, nullptr},
        {"https://api.github.com/repos/OWNER/REPO/contents/file2.txt", {}, nullptr},
        // 添加更多下载任务
    };

    // 为每个任务初始化easy_handle并添加到multi_handle
    for (auto& task : tasks) {
        task.easy_handle = curl_easy_init();
        if (!task.easy_handle) {
            std::cerr << "curl_easy_init() failed for URL: " << task.url << std::endl;
            continue;
        }

        struct curl_slist* slist = nullptr;
        slist = curl_slist_append(slist, gitToken.data());
        slist = curl_slist_append(slist, "Accept: application/vnd.github.v3.raw");

        curl_easy_setopt(task.easy_handle, CURLOPT_URL, task.url.c_str());
        curl_easy_setopt(task.easy_handle, CURLOPT_HTTPHEADER, slist);
        curl_easy_setopt(task.easy_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
        curl_easy_setopt(task.easy_handle, CURLOPT_WRITEDATA, &task.data);
        curl_easy_setopt(task.easy_handle, CURLOPT_USERAGENT, "curl/7.55.1");
        curl_easy_setopt(task.easy_handle, CURLOPT_FOLLOWLOCATION, 1L);
        curl_easy_setopt(task.easy_handle, CURLOPT_VERBOSE, 1L); // 调试用，发布时可关闭

        // 将easy_handle添加到multi_handle
        CURLcode res = curl_multi_add_handle(multi_handle, task.easy_handle);
        if (res != CURLE_OK) {
            std::cerr << "curl_multi_add_handle() failed: " << curl_easy_strerror(res) << std::endl;
            curl_easy_cleanup(task.easy_handle);
            task.easy_handle = nullptr;
            curl_slist_free_all(slist);
        }
    }

    // 处理并行下载
    int running_handles = 0;
    do {
        CURLMcode mc = curl_multi_perform(multi_handle, &running_handles);
        if (mc != CURLM_OK) {
            std::cerr << "curl_multi_perform() failed: " << curl_multi_strerror(mc) << std::endl;
            break;
        }

        // 检查是否有任务完成
        CURLMsg* msg = nullptr;
        int msgs_left = 0;
        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                // 找到对应的任务
                DownloadTask* completed_task = nullptr;
                for (auto& task : tasks) {
                    if (task.easy_handle == msg->easy_handle) {
                        completed_task = &task;
                        break;
                    }
                }

                if (completed_task) {
                    CURLcode res = msg->data.result;
                    OutputDebugStringA(curl_easy_strerror(res));
                    if (res != CURLE_OK) {
                        std::cerr << "Download failed for " << completed_task->url << ": " << curl_easy_strerror(res) << std::endl;
                    } else {
                        std::cout << "Download completed for " << completed_task->url << ", size: " << completed_task->data.size() << " bytes" << std::endl;
                        // 这里添加保存数据到磁盘的逻辑
                        // std::ofstream file("output.txt", std::ios::binary);
                        // file.write(completed_task->data.data(), completed_task->data.size());
                    }

                    // 清理该任务的资源
                    curl_multi_remove_handle(multi_handle, completed_task->easy_handle);
                    curl_easy_cleanup(completed_task->easy_handle);
                    completed_task->easy_handle = nullptr;

                    // 获取并释放HTTP头列表
                    struct curl_slist* slist = nullptr;
                    curl_easy_getinfo(completed_task->easy_handle, CURLINFO_HTTPHEADER, &slist);
                    if (slist) {
                        curl_slist_free_all(slist);
                    }
                }
            }
        }

        // 如果还有任务在运行，等待一段时间再轮询
        if (running_handles > 0) {
            curl_multi_wait(multi_handle, nullptr, 0, 1000, nullptr);
        }

    } while (running_handles > 0);

    // 清理multi_handle
    curl_multi_cleanup(multi_handle);
    // 全局清理
    curl_global_cleanup();

    return 0;
}

关键注意事项 #

• 替换代码中的YOUR_PRIVATE_GITHUB_TOKEN为你的实际GitHub私有Token
• 替换URL列表中的仓库路径和文件路径为实际目标
• curl_multi_wait用于在没有活动时阻塞等待，避免CPU空转，可根据需要调整等待时间（当前是1秒）
• 原代码中curl_global_cleanup被错误地放在单个请求的函数内，这会导致后续请求无法正常工作，修改后仅在程序结束时调用一次
• 每个任务的HTTP头列表需要在任务完成后释放，避免内存泄漏

内容的提问来源于stack exchange，提问作者Jabu