C语言中curl多句柄并行下载超时后重试5次的实现方法

要实现分块并行下载时的超时重试逻辑，核心是给每个下载分片维护独立的重试状态，并在非阻塞的multi事件循环中处理延迟重试，避免阻塞其他分片的下载。以下是具体实现步骤和代码示例：

1. 定义分片状态结构体 #

为每个下载分片定义结构体，跟踪重试次数、下次重试时间、下载范围等关键状态：

#include <curl/curl.h>
#include <stdio.h>
#include <time.h>
#include <stdlib.h>
#include <string.h>

#define MAX_RETRIES 5
#define RETRY_DELAY 60 // 重试间隔（秒）
#define LOW_SPEED_TIME 60L
#define LOW_SPEED_LIMIT 30L

typedef struct {
    CURL *curl;
    FILE *fp;
    char part_filename[256];
    long start;
    long end;
    int retry_count;
    time_t next_retry_time;
    int is_waiting_retry;
    int is_completed;
} DownloadPart;

2. 初始化分片下载句柄 #

为每个分片初始化curl句柄，设置分块下载的Range头、低速超时选项及数据写入回调：

// 数据写入回调：将下载内容写入分片文件
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
    return fwrite(ptr, size, nmemb, (FILE *)stream);
}

// 初始化单个分片的curl句柄
int init_part_curl(DownloadPart *part, const char *url) {
    part->curl = curl_easy_init();
    if (!part->curl) return -1;

    // 设置URL与分块范围
    curl_easy_setopt(part->curl, CURLOPT_URL, url);
    char range_header[64];
    snprintf(range_header, sizeof(range_header), "bytes=%ld-%ld", part->start, part->end);
    curl_easy_setopt(part->curl, CURLOPT_RANGE, range_header);

    // 设置低速超时规则
    curl_easy_setopt(part->curl, CURLOPT_LOW_SPEED_TIME, LOW_SPEED_TIME);
    curl_easy_setopt(part->curl, CURLOPT_LOW_SPEED_LIMIT, LOW_SPEED_LIMIT);

    // 打开分片文件并绑定写入回调
    part->fp = fopen(part->part_filename, "wb");
    if (!part->fp) {
        curl_easy_cleanup(part->curl);
        part->curl = NULL;
        return -1;
    }
    curl_easy_setopt(part->curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(part->curl, CURLOPT_WRITEDATA, part->fp);

    // 禁用信号，避免multi模式下的冲突
    curl_easy_setopt(part->curl, CURLOPT_NOSIGNAL, 1L);

    part->is_waiting_retry = 0;
    part->is_completed = 0;
    return 0;
}

3. 核心multi循环与重试逻辑 #

在主循环中处理完成的下载任务，判断是否触发重试；同时检查等待重试的分片，到时间后重新加入下载队列：

int main() {
    CURLM *multi_handle = curl_multi_init();
    if (!multi_handle) return 1;

    // 示例：创建3个分片（实际需根据目标文件大小计算分片范围）
    DownloadPart parts[3] = {
        {.start = 0, .end = 999999, .part_filename = "part1.tmp"},
        {.start = 1000000, .end = 1999999, .part_filename = "part2.tmp"},
        {.start = 2000000, .end = -1, .part_filename = "part3.tmp"} // 最后一个分片到文件末尾
    };
    int part_count = sizeof(parts) / sizeof(DownloadPart);

    // 初始化所有分片并加入multi句柄
    for (int i = 0; i < part_count; i++) {
        if (init_part_curl(&parts[i], "https://example.com/large-file.bin") == 0) {
            curl_multi_add_handle(multi_handle, parts[i].curl);
        } else {
            fprintf(stderr, "Failed to init part %d\n", i);
            return 1;
        }
    }

    int running_handles;
    while (1) {
        // 检查是否所有分片已完成
        int all_completed = 1;
        for (int i = 0; i < part_count; i++) {
            if (!parts[i].is_completed && !parts[i].is_waiting_retry) {
                all_completed = 0;
                break;
            }
        }
        if (all_completed) break;

        // 处理等待重试的分片：到时间则重新加入下载队列
        time_t now = time(NULL);
        for (int i = 0; i < part_count; i++) {
            if (parts[i].is_waiting_retry && now >= parts[i].next_retry_time) {
                // 清理旧句柄并重新初始化
                if (parts[i].curl) {
                    curl_easy_cleanup(parts[i].curl);
                }
                if (init_part_curl(&parts[i], "https://example.com/large-file.bin") == 0) {
                    parts[i].retry_count++;
                    parts[i].is_waiting_retry = 0;
                    curl_multi_add_handle(multi_handle, parts[i].curl);
                    printf("Retrying part %d, attempt %d\n", i, parts[i].retry_count);
                } else {
                    fprintf(stderr, "Failed to retry part %d\n", i);
                    parts[i].is_completed = 1;
                }
            }
        }

        // 执行multi下载操作
        CURLMcode mc = curl_multi_perform(multi_handle, &running_handles);
        if (mc != CURLM_OK) {
            fprintf(stderr, "curl_multi_perform failed: %s\n", curl_multi_strerror(mc));
            break;
        }

        // 无运行句柄时等待事件触发
        if (running_handles == 0) {
            curl_multi_wait(multi_handle, NULL, 0, 1000, NULL); // 等待1秒
            continue;
        }

        // 处理完成的下载任务
        CURLMsg *msg;
        int msgs_left;
        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                // 找到对应分片
                DownloadPart *current_part = NULL;
                for (int i = 0; i < part_count; i++) {
                    if (parts[i].curl == msg->easy_handle) {
                        current_part = &parts[i];
                        break;
                    }
                }
                if (!current_part) continue;

                // 关闭文件指针
                fclose(current_part->fp);
                current_part->fp = NULL;

                CURLcode res = msg->data.result;
                if (res == CURLE_OK) {
                    // 下载成功，标记完成
                    current_part->is_completed = 1;
                    printf("Part %s downloaded successfully\n", current_part->part_filename);
                } else if (res == CURLE_OPERATION_TIMEDOUT && current_part->retry_count < MAX_RETRIES) {
                    // 低速超时，触发重试
                    current_part->next_retry_time = time(NULL) + RETRY_DELAY;
                    current_part->is_waiting_retry = 1;
                    printf("Part %s timed out, will retry in %d seconds (attempt %d/%d)\n",
                           current_part->part_filename, RETRY_DELAY, current_part->retry_count + 1, MAX_RETRIES);
                } else {
                    // 其他错误或重试次数耗尽
                    fprintf(stderr, "Part %s failed: %s (retries exhausted)\n",
                            current_part->part_filename, curl_easy_strerror(res));
                    remove(current_part->part_filename); // 删除不完整分片
                    current_part->is_completed = 1;
                }

                // 从multi中移除并清理句柄
                curl_multi_remove_handle(multi_handle, current_part->curl);
                curl_easy_cleanup(current_part->curl);
                current_part->curl = NULL;
            }
        }

        // 等待事件，避免忙循环
        curl_multi_wait(multi_handle, NULL, 0, 500, NULL); // 等待500毫秒
    }

    // 合并分片（示例逻辑，可根据实际需求调整）
    printf("All parts downloaded, merging...\n");
    FILE *output = fopen("output.bin", "wb");
    if (output) {
        char buf[4096];
        size_t read_len;
        for (int i = 0; i < part_count; i++) {
            FILE *part_fp = fopen(parts[i].part_filename, "rb");
            if (part_fp) {
                while ((read_len = fread(buf, 1, sizeof(buf), part_fp)) > 0) {
                    fwrite(buf, 1, read_len, output);
                }
                fclose(part_fp);
                remove(parts[i].part_filename); // 删除临时分片
            }
        }
        fclose(output);
        printf("Merge completed, output saved to output.bin\n");
    }

    // 清理multi句柄
    curl_multi_cleanup(multi_handle);
    return 0;
}

关键细节说明 #

• 非阻塞重试：通过时间戳记录下次重试时间，在循环中定期检查，避免用sleep()阻塞整个程序，保证其他分片正常下载。
• 独立状态管理：每个分片的重试次数、等待状态独立维护，不会相互干扰。
• 资源安全：无论下载成功、失败还是重试，都正确关闭文件指针、清理curl句柄，避免内存泄漏。
• 续传优化：如果需要断点续传，可在初始化分片时检查临时文件大小，修改Range头为start + 文件大小到end，实现断点续传（示例中为简化逻辑，每次重试重新下载整个分片）。

内容的提问来源于stack exchange，提问作者steyn