如何解决Python requests下载Open Library大文件时卡顿无响应问题？

需要每月下载的Open Library数据转储文件包括：

• ol_dump_authors_latest.txt.gz
• ol_dump_works_latest.txt.gz
• ol_dump_editions_latest.txt.gz

其中works和editions文件体积较大，下载时会出现无报错但卡顿停止的情况，表现为文件大小长时间不再变化。你尝试的两种requests写法均未处理传输停滞、超时及断点续传逻辑，这是问题的核心所在。

首次尝试代码 #

dump_url = "https://openlibrary.org/data/ol_dump_editions_latest.txt.gz"
dump_path =  "temp_file/ol_dump_editions_latest.txt.gz"
session = requests.Session()
    with session.get(dump_url, stream=True) as r:
        r.raise_for_status()
        with open(dump_path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024*1024): 
                f.write(chunk)

第二次尝试代码 #

dump_url = "https://openlibrary.org/data/ol_dump_editions_latest.txt.gz"
dump_path =  "temp_file/ol_dump_editions_latest.txt.gz"
session = requests.Session()
with session.get(dump_url, stream=True) as r:
    r.raise_for_status()
    with open(dump_path, 'wb') as f:
        shutil.copyfileobj(r.raw, f)

优化后的解决方案 #

以下代码添加了断点续传、超时监控、自动重试及完整性校验，彻底解决大文件下载卡顿问题：

import requests
import os
import time
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def download_large_file(url, save_path, chunk_size=1024*1024, max_retries=3, timeout=30):
    # 配置会话重试策略
    session = requests.Session()
    retry_strategy = Retry(
        total=max_retries,
        backoff_factor=1,
        status_forcelist=[429, 500, 502, 503, 504]
    )
    adapter = HTTPAdapter(max_retries=retry_strategy)
    session.mount("https://", adapter)
    session.mount("http://", adapter)

    downloaded_size = 0
    # 断点续传：读取已下载文件大小
    if os.path.exists(save_path):
        downloaded_size = os.path.getsize(save_path)
        print(f"已下载 {downloaded_size/(1024*1024):.2f} MB，开始续传")

    try:
        headers = {"Range": f"bytes={downloaded_size}-"} if downloaded_size > 0 else {}
        with session.get(url, stream=True, headers=headers, timeout=timeout) as r:
            r.raise_for_status()

            # 获取总文件大小
            file_size = downloaded_size + int(r.headers.get("Content-Length", 0))
            if file_size:
                print(f"总文件大小: {file_size/(1024*1024):.2f} MB")

            with open(save_path, "ab") as f:
                for chunk in r.iter_content(chunk_size=chunk_size):
                    if chunk:
                        chunk_start = time.time()
                        f.write(chunk)
                        downloaded_size += len(chunk)
                        # 检测单块传输超时
                        if time.time() - chunk_start > timeout:
                            raise requests.exceptions.Timeout("单块传输超时")
                        # 实时打印进度
                        progress = (downloaded_size / file_size) * 100 if file_size else 0
                        print(f"\r进度: {progress:.1f}% ({downloaded_size/(1024*1024):.2f}/{file_size/(1024*1024):.2f} MB)", end="")

        # 校验文件完整性
        if file_size and os.path.getsize(save_path) != file_size:
            raise Exception(f"文件不完整，预期{file_size}字节，实际{os.path.getsize(save_path)}字节")
        
        print("\n下载完成")
    except Exception as e:
        print(f"\n下载失败: {str(e)}")
        raise

# 批量下载示例
if __name__ == "__main__":
    target_files = [
        "https://openlibrary.org/data/ol_dump_authors_latest.txt.gz",
        "https://openlibrary.org/data/ol_dump_works_latest.txt.gz",
        "https://openlibrary.org/data/ol_dump_editions_latest.txt.gz"
    ]
    save_directory = "temp_file"
    os.makedirs(save_directory, exist_ok=True)

    for file_url in target_files:
        filename = file_url.split("/")[-1]
        save_path = os.path.join(save_directory, filename)
        print(f"\n开始下载: {filename}")
        download_large_file(file_url, save_path)

关键优化点说明 #

1. 断点续传：通过Range请求头实现，中断后无需重新下载整个文件
2. 超时控制：全局请求超时+单块传输超时，避免连接停滞无限等待
3. 自动重试：对5xx错误、限流等场景自动重试，提升稳定性
4. 进度监控：实时显示下载进度，便于跟踪状态
5. 完整性校验：下载完成后对比文件大小，确保文件完整

内容的提问来源于stack exchange，提问作者Edin.A