如何并行读取文件并流式生成Zip包传输给客户端？

核心思路 #

DotNetZip（Ionic.Zip）库内置并行压缩支持，能在流式生成Zip包的同时，并行处理多个大文件的压缩，完全符合你的需求：无内存压力、无需本地临时文件、实时流式传输到客户端。

实现代码 #

首先安装DotNetZip NuGet包：

Install-Package DotNetZip

或通过.NET CLI：

dotnet add package DotNetZip

替换原有代码为：

using Ionic.Zip;
using Microsoft.AspNetCore.Mvc;
using System.IO;

public IActionResult DownloadZip(string[] arrLocalFilesPath)
{
    // 设置响应头，告知客户端接收Zip文件
    Response.Headers.Add("Content-Disposition", "attachment; filename=\"archive.zip\"");
    Response.ContentType = "application/zip";

    using var zipFile = new ZipFile();
    // 启用多文件并行压缩
    zipFile.ParallelCompression = true;
    // 设置压缩级别（对应原代码的Optimal）
    zipFile.CompressionLevel = Ionic.Zlib.CompressionLevel.BestCompression;
    // 禁用本地临时文件，强制流式处理
    zipFile.EnableZip64 = Zip64Option.AsNecessary; // 自动支持4GB以上大文件
    zipFile.TempFileFolder = null;

    // 添加所有需要压缩的文件
    foreach (var filePath in arrLocalFilesPath)
    {
        // 第二个参数为Zip内的目录路径，空值表示直接放在根目录
        zipFile.AddFile(filePath, "");
    }

    // 边压缩边写入响应流，实现实时传输
    zipFile.Save(Response.BodyWriter.AsStream());

    return new EmptyResult();
}

方案优势 #

• 并行压缩：通过ParallelCompression = true启用多文件并行处理，充分利用CPU资源提升大文件压缩效率
• 流式传输：直接将压缩后的Zip数据写入Response流，不会将整个Zip包加载到内存，无内存压力
• 无临时文件：禁用本地临时文件生成，全程在流中处理
• 大文件支持：自动适配超过4GB的大文件

若倾向于使用SharpZipLib，可通过管道流（PipeStream）实现并行压缩+顺序写入Zip流的方案，避免内存堆积。

实现代码 #

首先安装SharpZipLib NuGet包：

Install-Package SharpZipLib

或.NET CLI：

dotnet add package SharpZipLib

辅助类：统计压缩后的字节数

using System.IO;

public class CountingStream : Stream
{
    private readonly Stream _innerStream;
    public long BytesWritten { get; private set; }

    public CountingStream(Stream innerStream) => _innerStream = innerStream;

    public override bool CanRead => _innerStream.CanRead;
    public override bool CanSeek => _innerStream.CanSeek;
    public override bool CanWrite => _innerStream.CanWrite;
    public override long Length => _innerStream.Length;
    public override long Position { get => _innerStream.Position; set => _innerStream.Position = value; }

    public override void Flush() => _innerStream.Flush();
    public override int Read(byte[] buffer, int offset, int count) => _innerStream.Read(buffer, offset, count);
    public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin);
    public override void SetLength(long value) => _innerStream.SetLength(value);

    public override void Write(byte[] buffer, int offset, int count)
    {
        _innerStream.Write(buffer, offset, count);
        BytesWritten += count;
    }

    public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
    {
        await _innerStream.WriteAsync(buffer, offset, count, cancellationToken);
        BytesWritten += count;
    }
}

核心业务代码

using ICSharpCode.SharpZipLib.Checksums;
using ICSharpCode.SharpZipLib.Core;
using ICSharpCode.SharpZipLib.Zip;
using Microsoft.AspNetCore.Mvc;
using System.IO.Pipelines;

public async Task<IActionResult> DownloadZip(string[] arrLocalFilesPath)
{
    Response.Headers.Add("Content-Disposition", "attachment; filename=\"archive.zip\"");
    Response.ContentType = "application/zip";

    var responseStream = Response.BodyWriter.AsStream();
    using var zipOutputStream = new ZipOutputStream(responseStream);
    zipOutputStream.SetLevel(9); // 对应Optimal压缩级别
    zipOutputStream.UseZip64 = UseZip64.Off; // 需支持4GB+文件改为UseZip64.On

    // 并行启动所有文件的压缩任务，压缩后的数据写入管道流
    var compressionTasks = arrLocalFilesPath.Select(async filePath =>
    {
        var pipe = new Pipe();
        var fileName = Path.GetFileName(filePath);
        var countingStream = new CountingStream(pipe.Writer.AsStream());

        // 获取原始文件大小
        long originalSize;
        using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 8192, true))
        {
            originalSize = fileStream.Length;
        }

        // 计算CRC并压缩文件到管道
        var crc = new Crc32();
        using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 8192, true);
        using var crcStream = new CrcCalculatingStream(crc, fileStream);
        using var deflateStream = new DeflateStream(countingStream, CompressionLevel.Optimal, leaveOpen: true);

        await crcStream.CopyToAsync(deflateStream, HttpContext.RequestAborted);
        await deflateStream.FlushAsync(HttpContext.RequestAborted);
        await pipe.Writer.CompleteAsync();

        return new
        {
            FileName = fileName,
            PipeReader = pipe.Reader,
            OriginalSize = originalSize,
            Crc = crc.Value,
            CompressedSize = countingStream.BytesWritten
        };
    }).ToList();

    // 等待所有压缩任务完成
    var compressionResults = await Task.WhenAll(compressionTasks);

    // 按顺序将压缩后的数据写入Zip流，保证Zip结构正确
    foreach (var result in compressionResults)
    {
        var zipEntry = new ZipEntry(result.FileName)
        {
            DateTime = DateTime.Now,
            Size = result.OriginalSize,
            CompressedSize = result.CompressedSize,
            Crc = result.Crc,
            CompressionMethod = CompressionMethod.Deflated
        };

        zipOutputStream.PutNextEntry(zipEntry);
        await result.PipeReader.CopyToAsync(zipOutputStream, HttpContext.RequestAborted);
        zipOutputStream.CloseEntry();
        await result.PipeReader.CompleteAsync();
    }

    zipOutputStream.Finish();
    return new EmptyResult();
}

方案说明 #

• 每个文件的压缩过程通过Task并行执行，压缩数据写入内存高效的PipeStream
• 主线程等待所有压缩任务完成后，按顺序将管道数据写入ZipOutputStream，保证Zip包结构合法
• 全程无内存堆积，压缩数据边生成边写入响应流

原代码问题分析 #

你之前的DotNetZip代码仅用ParallelDeflateOutputStream将单个文件压缩到本地临时文件，未构建Zip包的条目结构（如文件头、目录信息等），因此无法生成客户端可识别的Zip流。正确做法是使用ZipFile类管理整个Zip包构建，并启用并行压缩选项。

内容的提问来源于stack exchange，提问作者Codingwiz