.NET Core处理Blob大文件导出时内存不足问题求解

最近开发一个导出项目，核心流程是从Blob Storage获取文件，合并后压缩为Zip包再上传回Blob Storage。小量文件时运行正常性能良好，但处理大量文件时会崩溃——原因是运行环境仅配备15GB内存，生成的Zip文件超出内存限制。试过Azure Functions，但该服务同样有15GB内存限制，问题依旧。

当前核心步骤及代码如下：

步骤1：批量下载Blob到内存字典 #

public async Task<Dictionary<string, byte[]>> DownloadManyAsync(Guid exportId)
{
    var tasks = new Queue<Task>();
    var files = new ConcurrentDictionary<string, byte[]>();

    var container = _blobServiceClient.GetBlobContainerClient("");
    var blobs = container.GetBlobs(prefix: "");
    var options = BlobStorageTools.GetOptions();


    foreach (var blob in blobs)
    {
        tasks.Enqueue(DownloadAndEnlist(container.GetBlobClient(blob.Name), files, options, exportId));
    }

    await Task.WhenAll(tasks);

    return files.ToDictionary(x => x.Key,
                              x => x.Value, 
                              files.Comparer);
}

public async Task DownloadAndEnlist(BlobClient blob, ConcurrentDictionary<string, byte[]> files, StorageTransferOptions options, Guid exportId)
{
    using var memoryStream = new MemoryStream();

    await blob.DownloadToAsync(memoryStream, default, options);

    files.TryAdd(blob.Name, memoryStream.ToArray());
}

步骤2：内存中生成完整Zip包 #

using var memoryStream = new MemoryTributary();

using (var archive = new ZipArchive(memoryStream, ZipArchiveMode.Create, true))
{
    for (int i = files.Count - 1; i >= 0; i--)
    {
        var file = files.ElementAt(i);

        var zipArchiveEntry = archive.CreateEntry(file.Key, CompressionLevel.Fastest);

        using var zipStream = zipArchiveEntry.Open();

        zipStream.Write(file.Value, 0, file.Value.Length);

        files.Remove(file.Key);
    }
}

步骤3：分块上传Zip到Blob #

public async Task<string> SaveExport(string fileName, Stream file)
{
    var cloudBlockBlob = _blobClient.GetContainerReference("").GetBlockBlobReference($"{fileName}.zip");

    BlockingCollection<string> blockList = new();
    Queue<Task> tasks = new();

    int bytesRead;
    int blockNumber = 0;


    if (file.Position != 0) file.Position = 0;

    do
    {
        blockNumber++;
        string blockId = $"{blockNumber:000000000}";
        string base64BlockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(blockId));

        byte[] buffer = new byte[8000000];
        bytesRead = await file.ReadAsync(buffer);

        tasks.Enqueue(Task.Run(async () =>
        {
            await cloudBlockBlob.PutBlockAsync(base64BlockId, new MemoryStream(buffer, 0, bytesRead) { Position = 0 }, null);

            blockList.Add(base64BlockId);
        }));
        
    } while (bytesRead == 8000000);

    await Task.WhenAll(tasks);

    await cloudBlockBlob.PutBlockListAsync(blockList);

    return cloudBlockBlob.Uri.ToString();
}

解决思路 #

1. 全流式处理（核心优化） #

问题根源是全量加载所有文件到内存+内存中生成完整Zip包，这会导致内存占用随文件总量线性增长。改为流式处理，全程不缓存全量数据：

直接将目标Blob的写入流作为ZipArchive的底层流，逐个下载Blob并写入Zip条目，写完即释放该文件内存，全程内存仅占用当前处理的文件块和压缩缓冲区。

示例代码：

public async Task<string> GenerateAndUploadZipAsync(Guid exportId)
{
    var container = _blobServiceClient.GetBlobContainerClient("your-container-name");
    var blobs = container.GetBlobs(prefix: "your-blob-prefix");
    var transferOptions = BlobStorageTools.GetOptions();

    // 创建目标Zip的Blob客户端
    var zipBlobClient = container.GetBlobClient($"{exportId}.zip");

    // 打开Blob的写入流，直接作为ZipArchive的底层流
    using var uploadStream = await zipBlobClient.OpenWriteAsync(overwrite: true);
    using var zipArchive = new ZipArchive(uploadStream, ZipArchiveMode.Create, leaveOpen: true);

    foreach (var blobItem in blobs)
    {
        var sourceBlobClient = container.GetBlobClient(blobItem.Name);
        // 创建Zip条目
        var zipEntry = zipArchive.CreateEntry(blobItem.Name, CompressionLevel.Fastest);
        
        // 直接将Blob内容流式写入Zip条目，不缓存到内存
        using var entryStream = zipEntry.Open();
        await sourceBlobClient.DownloadToAsync(entryStream, cancellationToken: default, transferOptions);
    }

    return zipBlobClient.Uri.ToString();
}

2. 单个大文件的分块流式处理 #

如果存在单个超大Blob，可以进一步对单个文件分块下载、分块写入Zip，避免加载整个大文件到内存：

public async Task StreamBlobToZipEntry(BlobClient blobClient, ZipArchiveEntry zipEntry, StorageTransferOptions options)
{
    using var entryStream = zipEntry.Open();
    var downloadResult = await blobClient.DownloadStreamingAsync(cancellationToken: default, options);
    
    // 分块读取Blob内容，每块8MB
    byte[] buffer = new byte[8 * 1024 * 1024];
    int bytesRead;
    using var contentStream = downloadResult.Content;
    
    while ((bytesRead = await contentStream.ReadAsync(buffer)) > 0)
    {
        await entryStream.WriteAsync(buffer.AsMemory(0, bytesRead));
        // 清空缓冲区释放内存（可选，视内存压力调整）
        Array.Clear(buffer, 0, bytesRead);
    }
}

3. 简化Blob上传逻辑 #

原代码手动管理PutBlock和BlockList，Azure.Storage.Blobs库的OpenWriteAsync和UploadStreamAsync会自动处理分块上传，无需手动管理，代码更简洁且不易出错。

4. 移除不必要的内存缓存 #

原流程中用ConcurrentDictionary缓存所有文件的byte[]是最大的内存消耗点，流式处理完全不需要这个结构，直接遍历Blob逐个处理即可，彻底消除这部分内存开销。

内容的提问来源于stack exchange，提问作者bini