求助：如何从URL下载.gz文件并解压后存储至S3？附现有代码

首先安装所需依赖：pip install requests boto3

方案一：流式处理（无需本地存储） #

直接流式下载.gz文件、解压后上传到S3，节省磁盘空间：

import requests
import gzip
import boto3
from io import BytesIO

# 替换为你的实际配置
S3_BUCKET_NAME = "你的S3桶名"
S3_TARGET_KEY = "解压后在S3的文件名.txt"

def download_uncompress_upload(url):
    with requests.get(url, stream=True) as r:
        r.raise_for_status()  # 检查请求是否成功
        # 将响应内容转为字节流用于解压
        with BytesIO(r.content) as gz_buffer:
            with gzip.GzipFile(fileobj=gz_buffer, mode='rb') as uncompressed_file:
                # 初始化S3客户端
                s3 = boto3.client('s3')
                # 直接上传解压后的内容到S3
                s3.upload_fileobj(uncompressed_file, S3_BUCKET_NAME, S3_TARGET_KEY)
                print(f"文件已上传至: s3://{S3_BUCKET_NAME}/{S3_TARGET_KEY}")

# 调用示例（需补充完整的.gz文件URL）
TARGET_URL = "https://api.botify.com/v1/jobs/具体的文件名.gz"
download_uncompress_upload(TARGET_URL)

方案二：先存本地再处理 #

如果需要保留本地文件，可使用此方式：

import os
import requests
import shutil
import gzip
import boto3

# 替换为你的实际配置
LOCAL_STORAGE_FOLDER = "本地存储文件夹路径"
S3_BUCKET_NAME = "你的S3桶名"
S3_TARGET_KEY = "解压后在S3的文件名.txt"

def download_file(url, folder_name):
    local_filename = url.split('/')[-1]
    path = os.path.join(folder_name, local_filename)
    # 自动创建本地文件夹
    os.makedirs(folder_name, exist_ok=True)
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(path, 'wb') as f:
            shutil.copyfileobj(r.raw, f)
    return path

def uncompress_upload(gz_file_path):
    # 生成解压后的文件名
    uncompressed_name = os.path.splitext(os.path.basename(gz_file_path))[0]
    uncompressed_path = os.path.join(os.path.dirname(gz_file_path), uncompressed_name)
    
    # 解压文件
    with gzip.open(gz_file_path, 'rb') as gz_file:
        with open(uncompressed_path, 'wb') as out_file:
            shutil.copyfileobj(gz_file, out_file)
    
    # 上传到S3
    s3 = boto3.client('s3')
    s3.upload_file(uncompressed_path, S3_BUCKET_NAME, S3_TARGET_KEY)
    print(f"文件已上传至: s3://{S3_BUCKET_NAME}/{S3_TARGET_KEY}")
    
    # 可选：删除本地临时文件
    os.remove(gz_file_path)
    os.remove(uncompressed_path)

# 调用示例
TARGET_URL = "https://api.botify.com/v1/jobs/具体的文件名.gz"
local_gz_path = download_file(TARGET_URL, LOCAL_STORAGE_FOLDER)
uncompress_upload(local_gz_path)

注意事项 #

• 确保AWS凭证已正确配置（可通过环境变量、~/.aws/credentials文件或IAM角色）
• 替换代码中所有占位的配置信息为实际值
• 确认目标S3桶存在，且你的AWS账号拥有该桶的上传权限
• 原URL需指向具体的.gz文件，当前给出的https://api.botify.com/v1/jobs/是目录地址，需补充完整文件名

内容的提问来源于stack exchange，提问作者marie20