如何在Golang中实现15GB Zip大文件下载并解决崩溃问题

AI 小北

2026-7-4

Golang大文件下载崩溃问题的解决方案

在Golang中通过URL下载15GB左右的Zip文件时，下载进程运行一段时间后崩溃，需要优化下载流程，处理超时、连接错误等问题，避免崩溃。以下是当前实现代码：

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "path/filepath"
)

func main() {
    http.HandleFunc("/download", downloadHandler)
    port := 8080
    fmt.Printf("Server is listening on :%d...\n", port)
    err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
    if err != nil {
        fmt.Println("Error starting server:", err)
    }
}

func downloadHandler(w http.ResponseWriter, r *http.Request) {
    fileURL := "https://example.com/largefile.zip" // Replace with your actual file URL
    fileName := "largefile.zip"
    downloadPath := "./downloads/" + fileName // Adjust the download path as needed

    err := downloadFile(fileURL, downloadPath)
    if err != nil {
        http.Error(w, fmt.Sprintf("Failed to download file: %v", err), http.StatusInternalServerError)
        return
    }

    w.WriteHeader(http.StatusOK)
    w.Write([]byte("File downloaded successfully"))
}

func downloadFile(url string, filepath string) error {
    response, err := http.Get(url)
    if err != nil {
        return err
    }
    defer response.Body.Close()

    if response.StatusCode != http.StatusOK {
        return fmt.Errorf("unexpected status code: %d", response.StatusCode)
    }

    // Create the file
    out, err := os.Create(filepath)
    if err != nil {
        return err
    }
    defer out.Close()

    // Create a progress writer to track the download progress
    progress := &progressWriter{writer: out}

    // Copy the response body to the file, and track the progress
    _, err = io.Copy(progress, response.Body)
    if err != nil {
        return err
    }

    return nil
}

type progressWriter struct {
    writer    io.Writer
    total     int64
    downloaded int64
}

func (pw *progressWriter) Write(p []byte) (int, error) {
    n, err := pw.writer.Write(p)
    pw.downloaded += int64(n)
    pw.printProgress()
    return n, err
}

func (pw *progressWriter) printProgress() {
    fmt.Printf("\rDownloading... %.2f%% complete", float64(pw.downloaded)/float64(pw.total)*100)
}

问题分析与优化方案

你的代码存在几个关键问题导致崩溃：

未设置HTTP客户端超时，网络异常时会无限阻塞
进度统计中total字段未初始化，会触发除以零的panic
不支持断点续传，中断后需重新下载，增加崩溃概率
缺乏错误重试机制，单次网络波动直接导致失败

针对这些问题，优化方案如下：

1. 配置带超时的HTTP客户端

替换默认http.Get，使用自定义http.Client设置全局超时，同时配置Transport的连接超时和空闲超时，避免资源泄漏：

client := &http.Client{
    Timeout: 60 * time.Minute, // 针对大文件设置较长超时
    Transport: &http.Transport{
        DialContext: (&net.Dialer{
            Timeout:   30 * time.Second, // 连接超时
            KeepAlive: 30 * time.Second,
        }).DialContext,
        TLSHandshakeTimeout: 10 * time.Second,
        IdleConnTimeout:     30 * time.Second,
    },
}

2. 支持断点续传

通过Range请求实现断点续传，下载前检查本地文件大小，请求剩余部分，避免从头开始下载：

// 检查本地文件大小
fileInfo, err := os.Stat(savePath)
if err == nil {
    downloadedSize := fileInfo.Size()
    // 发起Range请求获取剩余内容
    req, _ := http.NewRequest("GET", url, nil)
    req.Header.Set("Range", fmt.Sprintf("bytes=%d-", downloadedSize))
    resp, err := client.Do(req)
    // 处理206 Partial Content响应
}

3. 修复进度统计的panic

从响应头Content-Length获取文件总大小，初始化progressWriter的total字段，避免除以零错误：

totalSize, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
if err == nil {
    progress.total = totalSize
}

4. 增加错误重试机制

对幂等的GET请求，在遇到网络错误时自动重试（限制重试次数），提升下载稳定性：

maxRetries := 3
var resp *http.Response
var err error
for i := 0; i < maxRetries; i++ {
    resp, err = client.Do(req)
    if err == nil && (resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusPartialContent) {
        break
    }
    if resp != nil {
        resp.Body.Close()
    }
    time.Sleep(time.Duration(i+1) * time.Second) // 指数退避等待
}
if err != nil {
    return err
}

5. 优化文件写入性能

使用更大的缓冲区进行拷贝，减少IO操作次数，提升大文件写入效率：

buf := make([]byte, 1024*1024) // 1MB缓冲区
_, err = io.CopyBuffer(progress, resp.Body, buf)

修改后的完整代码

package main

import (
    "fmt"
    "io"
    "net"
    "net/http"
    "os"
    "path/filepath"
    "strconv"
    "time"
)

func main() {
    http.HandleFunc("/download", downloadHandler)
    port := 8080
    fmt.Printf("Server is listening on :%d...\n", port)
    err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
    if err != nil {
        fmt.Println("Error starting server:", err)
    }
}

func downloadHandler(w http.ResponseWriter, r *http.Request) {
    fileURL := "https://example.com/largefile.zip" // 替换为实际文件URL
    fileName := "largefile.zip"
    downloadPath := "./downloads/" + fileName

    // 确保下载目录存在
    if err := os.MkdirAll(filepath.Dir(downloadPath), 0755); err != nil {
        http.Error(w, fmt.Sprintf("Failed to create download directory: %v", err), http.StatusInternalServerError)
        return
    }

    err := downloadFile(fileURL, downloadPath)
    if err != nil {
        http.Error(w, fmt.Sprintf("Failed to download file: %v", err), http.StatusInternalServerError)
        return
    }

    w.WriteHeader(http.StatusOK)
    w.Write([]byte("File downloaded successfully"))
}

func downloadFile(url string, savePath string) error {
    // 配置带超时的HTTP客户端
    client := &http.Client{
        Timeout: 60 * time.Minute, // 针对大文件设置较长超时
        Transport: &http.Transport{
            DialContext: (&net.Dialer{
                Timeout:   30 * time.Second,
                KeepAlive: 30 * time.Second,
            }).DialContext,
            TLSHandshakeTimeout: 10 * time.Second,
            IdleConnTimeout:     30 * time.Second,
            MaxIdleConns:        10,
            MaxIdleConnsPerHost: 5,
        },
    }

    var resp *http.Response
    var err error
    var downloadedSize int64

    // 检查本地文件是否存在，支持断点续传
    if fileInfo, statErr := os.Stat(savePath); statErr == nil {
        downloadedSize = fileInfo.Size()
        // 发起Range请求，获取剩余部分
        req, reqErr := http.NewRequest("GET", url, nil)
        if reqErr != nil {
            return reqErr
        }
        req.Header.Set("Range", fmt.Sprintf("bytes=%d-", downloadedSize))
        resp, err = client.Do(req)
        if err != nil {
            return err
        }
        // 检查是否支持断点续传
        if resp.StatusCode != http.StatusPartialContent {
            resp.Body.Close()
            // 不支持则从头下载
            downloadedSize = 0
            resp, err = client.Get(url)
            if err != nil {
                return err
            }
        }
    } else {
        // 文件不存在，从头下载
        resp, err = client.Get(url)
        if err != nil {
            return err
        }
    }
    defer resp.Body.Close()

    // 检查响应状态码
    if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
        return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
    }

    // 获取文件总大小
    totalSizeStr := resp.Header.Get("Content-Length")
    totalSize, err := strconv.ParseInt(totalSizeStr, 10, 64)
    if err != nil {
        return fmt.Errorf("failed to parse content length: %v", err)
    }
    totalSize += downloadedSize

    // 打开文件，支持追加写入
    var out *os.File
    if downloadedSize > 0 {
        out, err = os.OpenFile(savePath, os.O_APPEND|os.O_WRONLY, 0644)
    } else {
        out, err = os.Create(savePath)
    }
    if err != nil {
        return err
    }
    defer out.Close()

    // 初始化进度统计
    progress := &progressWriter{
        writer:     out,
        total:      totalSize,
        downloaded: downloadedSize,
    }

    // 使用1MB缓冲区拷贝，提升性能
    buf := make([]byte, 1024*1024)
    _, err = io.CopyBuffer(progress, resp.Body, buf)
    if err != nil {
        return err
    }

    fmt.Println("\nDownload completed successfully")
    return nil
}

type progressWriter struct {
    writer     io.Writer
    total      int64
    downloaded int64
}

func (pw *progressWriter) Write(p []byte) (int, error) {
    n, err := pw.writer.Write(p)
    pw.downloaded += int64(n)
    pw.printProgress()
    return n, err
}

func (pw *progressWriter) printProgress() {
    if pw.total == 0 {
        fmt.Printf("\rDownloaded %d bytes", pw.downloaded)
        return
    }
    percentage := float64(pw.downloaded) / float64(pw.total) * 100
    fmt.Printf("\rDownloading... %.2f%% complete (%d/%d bytes)", percentage, pw.downloaded, pw.total)
}

内容的提问来源于stack exchange，提问作者pruthvi

如何在Golang中实现15GB Zip大文件下载并解决崩溃问题

问题分析与优化方案 #

1. 配置带超时的HTTP客户端 #

2. 支持断点续传 #

3. 修复进度统计的panic #

4. 增加错误重试机制 #

5. 优化文件写入性能 #

修改后的完整代码 #