基于React与Express实现分片并行可续传单文件上传的方案问询

核心逻辑说明 #

实现断点续传的关键是给每个文件生成唯一标识（基于文件内容的MD5哈希），前端上传前先查询后端已上传的分片，只传剩余部分；后端按标识独立存储分片，全部接收完成后按顺序合并为原始文件。

前端（React）实现 #

1. 依赖安装 #

npm install spark-md5 axios

spark-md5用于计算文件内容哈希，axios处理HTTP请求。

2. 核心代码实现 #

import { useState } from 'react';
import SparkMD5 from 'spark-md5';
import axios from 'axios';

const ChunkUpload = () => {
  const [progress, setProgress] = useState(0);
  const [uploading, setUploading] = useState(false);
  // 分片大小：10MB，可根据网络情况调整
  const CHUNK_SIZE = 10 * 1024 * 1024;
  // 最大并发上传数
  const MAX_CONCURRENT = 6;

  // 计算文件唯一哈希（断点续传的核心标识）
  const calculateFileHash = async (file) => {
    return new Promise((resolve) => {
      const reader = new FileReader();
      const spark = new SparkMD5.ArrayBuffer();
      reader.onload = (e) => {
        spark.append(e.target.result);
        resolve(spark.end());
      };
      reader.readAsArrayBuffer(file);
    });
  };

  // 查询后端已上传的分片索引
  const getUploadedChunks = async (fileHash) => {
    const res = await axios.get('/api/upload/check', { params: { fileHash } });
    return res.data.uploadedChunks || [];
  };

  // 上传单个分片
  const uploadChunk = async (chunk, index, fileHash, fileName) => {
    const formData = new FormData();
    formData.append('chunk', chunk);
    formData.append('index', index);
    formData.append('fileHash', fileHash);
    formData.append('fileName', fileName);
    await axios.post('/api/upload/chunk', formData, {
      headers: { 'Content-Type': 'multipart/form-data' },
    });
  };

  // 并发控制上传剩余分片
  const uploadChunks = async (chunks, fileHash, fileName, uploadedChunks) => {
    let uploadedCount = uploadedChunks.length;
    const totalChunks = chunks.length;
    const pool = [];

    for (let i = 0; i < totalChunks; i++) {
      // 跳过已上传的分片
      if (uploadedChunks.includes(i)) continue;

      // 控制并发数，超过6个则等待最早完成的任务
      if (pool.length >= MAX_CONCURRENT) {
        await Promise.race(pool);
      }

      const task = uploadChunk(chunks[i], i, fileHash, fileName)
        .then(() => {
          uploadedCount++;
          setProgress(Math.floor((uploadedCount / totalChunks) * 100));
          // 从并发池移除已完成的任务
          const idx = pool.indexOf(task);
          if (idx !== -1) pool.splice(idx, 1);
        })
        .catch((err) => {
          console.error(`分片${i}上传失败:`, err);
          throw err; // 可改为自动重试逻辑
        });

      pool.push(task);
    }

    // 等待所有剩余任务完成
    await Promise.allSettled(pool);
  };

  // 通知后端合并分片
  const mergeChunks = async (fileHash, fileName, totalChunks) => {
    await axios.post('/api/upload/merge', {
      fileHash,
      fileName,
      totalChunks,
    });
  };

  // 主上传流程
  const handleFileUpload = async (e) => {
    const file = e.target.files[0];
    if (!file) return;

    setUploading(true);
    setProgress(0);

    try {
      // 1. 计算文件哈希
      const fileHash = await calculateFileHash(file);
      // 2. 查询已上传分片
      const uploadedChunks = await getUploadedChunks(fileHash);
      // 3. 分割文件为分片
      const chunks = [];
      const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
      for (let i = 0; i < totalChunks; i++) {
        const start = i * CHUNK_SIZE;
        const end = Math.min(start + CHUNK_SIZE, file.size);
        chunks.push(file.slice(start, end));
      }
      // 4. 上传剩余分片
      await uploadChunks(chunks, fileHash, file.name, uploadedChunks);
      // 5. 合并分片
      await mergeChunks(fileHash, file.name, totalChunks);
      alert('文件上传完成！');
    } catch (err) {
      console.error('上传失败:', err);
      alert('上传失败，请稍后重试');
    } finally {
      setUploading(false);
    }
  };

  return (
    <div>
      <input
        type="file"
        onChange={handleFileUpload}
        disabled={uploading}
      />
      {uploading && <div>上传进度：{progress}%</div>}
    </div>
  );
};

export default ChunkUpload;

后端（Express）实现 #

1. 依赖安装 #

npm install express multer fs-extra

multer处理文件上传，fs-extra简化异步文件操作。

2. 核心代码实现 #

const express = require('express');
const multer = require('multer');
const fs = require('fs-extra');
const path = require('path');

const app = express();
app.use(express.json());

// 存储目录配置
const UPLOAD_DIR = path.resolve(__dirname, './uploads');
const TEMP_DIR = path.resolve(UPLOAD_DIR, './temp');
// 确保目录存在
fs.ensureDirSync(UPLOAD_DIR);
fs.ensureDirSync(TEMP_DIR);

// multer配置：分片存储到对应文件哈希的临时目录
const storage = multer.diskStorage({
  destination: async (req, file, cb) => {
    const { fileHash } = req.body;
    const chunkDir = path.resolve(TEMP_DIR, fileHash);
    await fs.ensureDir(chunkDir);
    cb(null, chunkDir);
  },
  filename: (req, file, cb) => {
    // 用分片索引作为文件名，保证合并顺序
    cb(null, req.body.index);
  },
});
const upload = multer({ storage });

// 接口1：查询已上传分片
app.get('/api/upload/check', async (req, res) => {
  try {
    const { fileHash } = req.query;
    const chunkDir = path.resolve(TEMP_DIR, fileHash);
    let uploadedChunks = [];
    if (await fs.pathExists(chunkDir)) {
      // 获取所有分片文件名并转为数字数组排序
      uploadedChunks = await fs.readdir(chunkDir);
      uploadedChunks = uploadedChunks.map(Number).sort((a, b) => a - b);
    }
    res.json({ code: 0, uploadedChunks });
  } catch (err) {
    res.status(500).json({ code: -1, msg: '查询失败' });
  }
});

// 接口2：接收分片
app.post('/api/upload/chunk', upload.single('chunk'), (req, res) => {
  try {
    res.json({ code: 0, msg: '分片上传成功' });
  } catch (err) {
    res.status(500).json({ code: -1, msg: '分片上传失败' });
  }
});

// 接口3：合并分片
app.post('/api/upload/merge', async (req, res) => {
  try {
    const { fileHash, fileName, totalChunks } = req.body;
    const chunkDir = path.resolve(TEMP_DIR, fileHash);
    const targetPath = path.resolve(UPLOAD_DIR, fileName);

    // 创建写入流合并分片
    const writeStream = fs.createWriteStream(targetPath);

    for (let i = 0; i < totalChunks; i++) {
      const chunkPath = path.resolve(chunkDir, String(i));
      const chunkBuffer = await fs.readFile(chunkPath);
      await new Promise((resolve) => writeStream.write(chunkBuffer, resolve));
      // 删除单个分片文件
      await fs.remove(chunkPath);
    }

    // 关闭流并删除临时目录
    await new Promise((resolve) => writeStream.end(resolve));
    await fs.remove(chunkDir);

    res.json({ code: 0, msg: '文件合并完成' });
  } catch (err) {
    res.status(500).json({ code: -1, msg: '文件合并失败' });
  }
});

app.listen(3001, () => {
  console.log('服务器运行在 http://localhost:3001');
});

关键细节补充 #

1. 哈希计算优化：大文件计算哈希可能卡顿，可改为分片计算哈希再合并结果，避免占用主线程。
2. 重试机制：前端可给分片上传添加重试逻辑（比如失败后重试3次），提升稳定性。
3. 临时文件清理：可添加定时任务，清理超过7天的未完成分片目录，避免存储空间浪费。
4. 文件名去重：后端合并时可给文件名添加哈希后缀，避免同名文件覆盖。

内容的提问来源于stack exchange，提问作者Syed Amaan Ali