使用WebClient.UploadValues分块上传需认证大文件的内存问题求助

核心思路 #

• 把大文件分割成固定大小的块（比如5MB，可根据服务器配置调整）
• 每个分块请求携带认证表单数据，同时传递分块索引、总块数、文件唯一标识（让服务端能正确拼接）
• 所有分块上传完成后，发送合并请求触发服务端拼接文件

客户端实现代码（WebClient版） #

替代WebClient.UploadValues，用分块方式避免内存溢出：

using System;
using System.IO;
using System.Net;
using System.Collections.Specialized;

public class ChunkedFileUploader
{
    // 分块大小，建议5-10MB，平衡内存占用和请求次数
    private const int ChunkSize = 1024 * 1024 * 5;

    public void UploadWithAuth(string uploadApiUrl, string localFilePath, NameValueCollection authFormData)
    {
        var fileInfo = new FileInfo(localFilePath);
        var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
        var fileUniqueId = Guid.NewGuid().ToString(); // 服务端用这个标识同一文件的分块

        using (var fileStream = new FileStream(localFilePath, FileMode.Open, FileAccess.Read))
        {
            for (int chunkIdx = 0; chunkIdx < totalChunks; chunkIdx++)
            {
                var currentChunkSize = Math.Min(ChunkSize, (int)(fileInfo.Length - chunkIdx * ChunkSize));
                var chunkBuffer = new byte[currentChunkSize];
                fileStream.Read(chunkBuffer, 0, currentChunkSize);

                // 组装当前分块的表单数据：原有认证信息+分块元数据
                var requestForm = new NameValueCollection(authFormData);
                requestForm.Add("FileId", fileUniqueId);
                requestForm.Add("ChunkIndex", chunkIdx.ToString());
                requestForm.Add("TotalChunks", totalChunks.ToString());
                requestForm.Add("FileName", fileInfo.Name);

                using (var client = new WebClient())
                {
                    client.QueryString.Add(requestForm);
                    // 上传当前分块的字节数据
                    var responseBytes = client.UploadData(uploadApiUrl, "POST", chunkBuffer);
                    var responseMsg = System.Text.Encoding.UTF8.GetString(responseBytes);
                    if (responseMsg != "CHUNK_OK")
                    {
                        throw new Exception($"分块 {chunkIdx+1}/{totalChunks} 上传失败，服务端返回：{responseMsg}");
                    }
                }
            }
        }

        // 所有分块上传完成，发送合并请求
        using (var client = new WebClient())
        {
            var mergeForm = new NameValueCollection(authFormData);
            mergeForm.Add("FileId", fileUniqueId);
            mergeForm.Add("FileName", fileInfo.Name);
            var mergeResponse = client.UploadValues($"{uploadApiUrl}/merge", "POST", mergeForm);
            var mergeMsg = System.Text.Encoding.UTF8.GetString(mergeResponse);
            if (mergeMsg != "FILE_MERGED")
            {
                throw new Exception("文件合并失败，服务端返回：" + mergeMsg);
            }
        }
    }
}

客户端实现代码（HttpClient版，更灵活） #

如果WebClient限制较多，用HttpClient做流式上传，内存占用更低：

using System;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using System.Collections.Generic;

public class HttpClientChunkedUploader
{
    private const int ChunkSize = 1024 * 1024 * 5;

    public async Task UploadWithAuthAsync(string uploadApiUrl, string localFilePath, Dictionary<string, string> authFormData)
    {
        var fileInfo = new FileInfo(localFilePath);
        var totalChunks = (int)Math.Ceiling((double)fileInfo.Length / ChunkSize);
        var fileUniqueId = Guid.NewGuid().ToString();

        using (var httpClient = new HttpClient())
        using (var fileStream = new FileStream(localFilePath, FileMode.Open, FileAccess.Read))
        {
            for (int chunkIdx = 0; chunkIdx < totalChunks; chunkIdx++)
            {
                var currentChunkSize = Math.Min(ChunkSize, (int)(fileInfo.Length - chunkIdx * ChunkSize));
                var chunkStream = new MemoryStream(currentChunkSize);
                await fileStream.CopyToAsync(chunkStream, currentChunkSize);
                chunkStream.Position = 0;

                // 构建多部分表单数据：认证信息+分块元数据+分块文件流
                var formContent = new MultipartFormDataContent();
                foreach (var kvp in authFormData)
                {
                    formContent.Add(new StringContent(kvp.Value), kvp.Key);
                }
                formContent.Add(new StringContent(fileUniqueId), "FileId");
                formContent.Add(new StringContent(chunkIdx.ToString()), "ChunkIndex");
                formContent.Add(new StringContent(totalChunks.ToString()), "TotalChunks");
                formContent.Add(new StringContent(fileInfo.Name), "FileName");
                formContent.Add(new StreamContent(chunkStream), "FileChunk", $"temp_chunk_{chunkIdx}");

                var response = await httpClient.PostAsync(uploadApiUrl, formContent);
                response.EnsureSuccessStatusCode();
                var responseMsg = await response.Content.ReadAsStringAsync();
                if (responseMsg != "CHUNK_OK")
                {
                    throw new Exception($"分块 {chunkIdx+1}/{totalChunks} 上传失败");
                }
            }

            // 发送合并请求
            var mergeForm = new FormUrlEncodedContent(authFormData);
            mergeForm.Add(new KeyValuePair<string, string>("FileId", fileUniqueId));
            mergeForm.Add(new KeyValuePair<string, string>("FileName", fileInfo.Name));
            var mergeResponse = await httpClient.PostAsync($"{uploadApiUrl}/merge", mergeForm);
            mergeResponse.EnsureSuccessStatusCode();
            var mergeMsg = await mergeResponse.Content.ReadAsStringAsync();
            if (mergeMsg != "FILE_MERGED")
            {
                throw new Exception("文件合并失败");
            }
        }
    }
}

关键注意事项 #

• 分块大小调整：根据服务器的最大请求限制和自身内存情况调整，避免单块过大导致内存压力，或过小导致请求过多。
• 服务端配合：服务端需要实现三个逻辑：接收分块并临时存储、校验分块完整性、接收合并请求后按顺序拼接所有分块。
• 断点续传优化：如果需要支持断点，客户端可以先请求服务端查询已上传的分块索引，跳过已完成的分块。
• 认证一致性：每个分块请求和合并请求都要携带完整的认证表单数据，确保服务端能验证身份。

内容的提问来源于stack exchange，提问作者user16626854