大文件分片上传两难：服务器请求压力与客户端卡顿

问题背景 #

我正在实现大文件分片上传功能，目前仅完成前端分片处理，还未对接完整上传逻辑。核心矛盾点：上传2GB文件时，若分片数设为1000+，大量请求会给服务器带来类DDoS压力；分片数低于100的话，客户端会出现严重卡顿。

选择分片上传的原因：

• 支持暂停上传
• 分片因连接问题失败时，可从失败分片处重试

现有代码 #

前端代码 #

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>Video Thumbnail Generator</title>
    <link
      href="https://cdn.jsdelivr.net/npm/daisyui@4.12.10/dist/full.min.css"
      rel="stylesheet"
      type="text/css"
    />
    <script src="https://cdn.tailwindcss.com"></script>
  </head>
  <body>
    <span class="loading loading-spinner loading-lg"></span>
    <h1>File Chunk Processor</h1>
    <input type="file" id="fileInput" />

    <button id="processButton" onclick="DoChunk()">Process File</button>

    <script>
      function DoChunk() {
        
        let chunkCount = 1700;
        let chunkIndex = 0;
        processNextChunk(chunkIndex, chunkCount);
      }

      function processNextChunk(chunkIndex, chunkCount) {
        if (chunkIndex < chunkCount) {
          processFileChunk('fileInput', chunkIndex, chunkCount, function () {
            setTimeout(() => {
              processNextChunk(chunkIndex + 1, chunkCount);
            }, 100); // Adjust the delay as needed
          });
        }
      }

      function processFileChunk(elementId, chunkIndex, chunkCount, callback) {
        // Get the file input element
        const inputElement = document.getElementById(elementId);

        // Check if the input element and file are available
        if (!inputElement || !inputElement.files || !inputElement.files[0]) {
          console.error('No file selected or element not found');
          return;
        }

        // Get the selected file
        const file = inputElement.files[0];

        // Calculate the size of each chunk
        const chunkSize = Math.ceil(file.size / chunkCount);
        const start = chunkIndex * chunkSize;
        const end = Math.min(start + chunkSize, file.size);

        // Create a Blob for the specific chunk
        const chunk = file.slice(start, end);

        // Create a FileReader to read the chunk
        const reader = new FileReader();

        reader.onload = function (event) {
          // Get the chunk content as a Base64 string
          const base64String = event.target.result.split(',')[1]; // Remove data URL part

          // Output or process the chunk as needed
          console.log(`Chunk ${chunkIndex + 1} of ${chunkCount}:`);
          console.log(base64String);

          if (callback) {
            callback();
          }
        };

        reader.onerror = function (error) {
          console.error('Error reading file chunk:', error);
          if (callback) {
            callback();
          }
        };

        // Read the chunk as a Data URL (Base64 string)
        reader.readAsDataURL(chunk);
      }
    </script>
  </body>
</html>

后端ASP.NET接口代码 #

public async Task<IActionResult> UploadChunkAsync([FromBody] FileChunkRequest request,
        CancellationToken cancellationToken = default)
    {
        var requestToken = _jwtTokenRepository.GetJwtToken();
        var loggedInUser = _jwtTokenRepository.ExtractUserDataFromToken(requestToken);

        var blobTableClient = _blobClientFactory.BlobTableClient(TableName.StashChunkDetail);

        var stashChunkDetail = blobTableClient
            .Query<StashChunkDetail>(x => x.RowKey == request.UploadToken && x.UserId == loggedInUser.id)
            .SingleOrDefault();

        if (stashChunkDetail != null)
        {
            var currentChunkSize = request.Data.SizeMB();
            var isSizeOutOfDeal = stashChunkDetail.TotalUploadedSizeMb + currentChunkSize >
                                  stashChunkDetail.FileSizeMb;
            var containerName = Enum.Parse<ContainerName>(stashChunkDetail.PartitionKey);

            if (isSizeOutOfDeal)
            {
                //delete table 
                // ReSharper disable once MethodSupportsCancellation
                await blobTableClient.DeleteEntityAsync(stashChunkDetail.PartitionKey, stashChunkDetail.RowKey);

                //delete commit data form blob storage
                await _fileUploadService.DeleteObjectAsync(containerName, stashChunkDetail.RowKey);

                return BadRequest(
                    $"the size of chunks is more than {stashChunkDetail.FileSizeMb} MB please request new upload token");
            }

            var isLast = (request.Data.SizeMB() + stashChunkDetail.TotalUploadedSizeMb) >=
                         stashChunkDetail.FileSizeMb;

            int totalUploadedChunks = stashChunkDetail.TotalUploadedChunks;
            int currentChunk;

            if (totalUploadedChunks == 0)
            {
                currentChunk = 0;
            }
            else
            {
                currentChunk = totalUploadedChunks + 1;
            }


            var fileChunkDto = new FileChunkDto()
            {
                FileFormat = stashChunkDetail.FileFormat,
                ContainerName = containerName,
                FileName = stashChunkDetail.RowKey,
                Data = request.Data,
                AccessTier = stashChunkDetail.AccessTier,
                CurrentChunk = currentChunk,
                TotalUploadedChunks = stashChunkDetail.TotalUploadedChunks
            };

            await _fileUploadService.UploadChunkAsync(fileChunkDto, cancellationToken);


            stashChunkDetail.TotalUploadedChunks += 1;
            stashChunkDetail.TotalUploadedSizeMb += request.Data.SizeMB();

            // ReSharper disable once MethodSupportsCancellation
            await blobTableClient.UpdateEntityAsync(stashChunkDetail, ETag.All);

            var responseChunkProgress = new ChunkUploadResponse()
            {
                TotalUploadedChunks = stashChunkDetail.TotalUploadedChunks,
                TotalUploadedSizeMB = stashChunkDetail.TotalUploadedSizeMb
            };

            return StatusCode(StatusCodes.Status201Created, responseChunkProgress);
        }

        return BadRequest("Please Request New Upload Token");
    }

解决办法 #

1. 固定分片大小，平衡两端压力 #

放弃按分片数计算，改用固定大小分片，比如设置为10MB。2GB文件会分成200个左右的分片，既不会让客户端因单分片过大卡顿，也不会让服务器承受过多请求。

2. 前端实现有限并发上传 #

当前串行上传效率低，全并发又会压垮服务器，建议采用3-5个并发请求的策略：

• 维护分片队列，同时启动指定数量的上传任务
• 某个上传任务完成后，自动从队列中取下一个分片继续上传
• 替换setTimeout串行逻辑，用Promise和async/await实现异步控制，避免UI卡顿

优化后的前端核心代码示例：

const CHUNK_SIZE = 10 * 1024 * 1024; // 10MB
const MAX_CONCURRENCY = 3; // 最大并发数

async function DoChunk() {
    const inputElement = document.getElementById('fileInput');
    if (!inputElement?.files?.[0]) return;
    const file = inputElement.files[0];
    const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
    let chunkIndex = 0;
    const activeRequests = [];

    while (chunkIndex < totalChunks) {
        // 控制并发数
        if (activeRequests.length < MAX_CONCURRENCY) {
            const promise = uploadChunk(file, chunkIndex, totalChunks);
            activeRequests.push(promise);
            // 请求完成后从活跃列表移除
            promise.then(() => {
                const index = activeRequests.indexOf(promise);
                if (index !== -1) activeRequests.splice(index, 1);
            });
            chunkIndex++;
        }
        // 等待任意一个请求完成
        await Promise.race(activeRequests);
    }
    // 等待所有剩余请求完成
    await Promise.all(activeRequests);
    console.log('所有分片上传完成');
}

async function uploadChunk(file, chunkIndex, totalChunks) {
    const start = chunkIndex * CHUNK_SIZE;
    const end = Math.min(start + CHUNK_SIZE, file.size);
    const chunk = file.slice(start, end);
    
    // 直接用FormData上传Blob，避免Base64编码
    const formData = new FormData();
    formData.append('chunk', chunk);
    formData.append('chunkIndex', chunkIndex);
    formData.append('totalChunks', totalChunks);
    formData.append('uploadToken', '你的上传令牌'); // 需要先从服务器获取

    try {
        const response = await fetch('/api/upload-chunk', {
            method: 'POST',
            body: formData,
            headers: {
                'Authorization': `Bearer ${你的JWT令牌}`
            }
        });
        const result = await response.json();
        console.log(`分片 ${chunkIndex + 1}/${totalChunks} 上传完成`, result);
    } catch (error) {
        console.error(`分片 ${chunkIndex + 1}/${totalChunks} 上传失败`, error);
        // 可添加重试逻辑
        throw error;
    }
}

3. 前端优化分片读取逻辑 #

• 去掉FileReader.readAsDataURL转Base64的步骤，直接上传Blob，减少30%的数据传输量，同时降低客户端编码耗时
• 使用fetch异步API替代回调，代码更易维护

4. 后端配套优化 #

• 增加单用户分片上传限流策略，比如限制每分钟最多50个请求，避免服务器压力过大
• 批量更新分片状态，减少数据库操作次数（比如每上传10个分片更新一次总状态）
• 采用异步合并分片策略，所有分片上传完成后，后台异步合并文件，提升接口响应速度

5. 完善断点续传逻辑 #

前端上传前先请求服务器，获取当前文件已上传的分片列表，跳过已上传的分片，避免重复上传，同时支持暂停后恢复。

内容的提问来源于stack exchange，提问作者Regestea