FastAPI大文件分片上传适配：CSV/Excel文件拆分上传需求

要解决大文件上传的问题，我们需要同时修改前端和后端代码：

• 前端：将文件按类型拆分（CSV按行拆分避免断行，XLS/XLSX按字节拆分），并携带分片元数据（分片索引、总分片数、原文件名）多次调用上传接口。
• 后端：接收分片并临时存储，待所有分片上传完成后合并为原文件，再上传到目标存储并清理临时文件。

更新FastAPI接口，新增分片元数据参数，实现分片存储与合并逻辑：

import os
import shutil
from fastapi import FastAPI, File, Form, status
from fastapi.responses import HTMLResponse
from datetime import datetime
# 假设 email_vask_file_drive 已导入

# 临时分片存储目录，确保服务器有写入权限
TEMP_DIR = "./temp_chunks"
os.makedirs(TEMP_DIR, exist_ok=True)

@app.post("/upload", status_code=status.HTTP_201_CREATED, response_class=HTMLResponse)
async def upload_file(
    cvr: str = Form(...),
    file: UploadFile = File(...),
    chunk_index: int = Form(...),
    total_chunks: int = Form(...),
    original_filename: str = Form(...)
):
    # 允许的文件类型
    allowed_types = [
        'text/csv',
        'application/vnd.ms-excel',
        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
    ]
    if file.content_type not in allowed_types:
        return HTMLResponse(content="<html><body>无效的文件类型</body></html>", status_code=400)
    
    # 创建唯一的上传目录（避免同CVR+文件名的并发上传冲突）
    upload_dir = os.path.join(TEMP_DIR, f"{cvr}_{original_filename.replace('/', '_')}_{datetime.now().timestamp()}")
    os.makedirs(upload_dir, exist_ok=True)
    
    # 保存当前分片
    chunk_path = os.path.join(upload_dir, f"chunk_{chunk_index}")
    with open(chunk_path, "wb") as f:
        shutil.copyfileobj(file.file, f)
    
    # 检查是否所有分片已上传
    uploaded_chunks = len([f for f in os.listdir(upload_dir) if f.startswith("chunk_")])
    if uploaded_chunks == total_chunks:
        merged_filename = f"{cvr}_{datetime.date.today().strftime('%d_%m_%Y')}_{original_filename}"
        merged_path = os.path.join(TEMP_DIR, merged_filename)
        
        try:
            # 合并所有分片
            with open(merged_path, "wb") as merged_file:
                for i in range(total_chunks):
                    chunk_path = os.path.join(upload_dir, f"chunk_{i}")
                    with open(chunk_path, "rb") as chunk_file:
                        merged_file.write(chunk_file.read())
            
            # 上传到目标存储
            with open(merged_path, "rb") as f:
                res = email_vask_file_drive.put(merged_filename, f)
            
            # 清理临时文件
            shutil.rmtree(upload_dir)
            os.remove(merged_path)
            
            return HTMLResponse(content="<html><body>文件上传并合并成功</body></html>")
        except Exception as e:
            # 处理上传失败的情况，可添加日志记录
            return HTMLResponse(content=f"<html><body>合并文件上传失败：{str(e)}</body></html>", status_code=500)
    else:
        return HTMLResponse(content=f"<html><body>分片 {chunk_index+1}/{total_chunks} 上传成功</body></html>")

替换原表单的默认提交行为，实现文件分片上传逻辑：

<form id="uploadForm" method="post" action="https://my_url/upload" enctype="multipart/form-data">
    <label for="name" id="firmanavn">客户编号:</label>
    <input type="text" class="num-only" id="name" name="cvr" maxlength="8" pattern="\d{8}"
        oninvalid="this.setCustomValidity('请输入8位数字')"
        oninput="this.setCustomValidity('')" required>
    <input id="fileupload" type="file" name="file"
        accept=".csv,.xls,.xlsx,text/csv,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.ms-excel"
        required />
    <input type="submit" id="upload" value="上传文件">
</form>

<script>
const form = document.getElementById('uploadForm');
const fileInput = document.getElementById('fileupload');
const chunkSize = 5 * 1024 * 1024; // 5MB分片大小

form.addEventListener('submit', async (e) => {
    e.preventDefault();
    
    const cvr = document.getElementById('name').value;
    const file = fileInput.files[0];
    if (!file) return;
    
    const totalChunks = Math.ceil(file.size / chunkSize);
    const originalFilename = file.name;
    const contentType = file.type;
    
    // 处理CSV文件：按行拆分避免断行
    let csvChunks = [];
    if (contentType === 'text/csv') {
        const textContent = await file.text();
        const lines = textContent.split('\n');
        let currentChunk = [];
        let currentSize = 0;
        
        // 首分片保留表头
        if (lines.length > 0) {
            currentChunk.push(lines[0]);
            currentSize += lines[0].length + 1; // +1 换行符长度
        }
        
        for (let i = 1; i < lines.length; i++) {
            const lineSize = lines[i].length + 1;
            // 当前分片加新行超过大小限制时，保存当前分片并重置
            if (currentSize + lineSize > chunkSize && currentChunk.length > 0) {
                csvChunks.push(currentChunk.join('\n'));
                currentChunk = [];
                currentSize = 0;
            }
            currentChunk.push(lines[i]);
            currentSize += lineSize;
        }
        // 保存最后一个分片
        if (currentChunk.length > 0) csvChunks.push(currentChunk.join('\n'));
    }
    
    // 逐个上传分片
    for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
        let chunk;
        
        if (contentType === 'text/csv') {
            chunk = new Blob([csvChunks[chunkIndex]], { type: contentType });
        } else {
            // 二进制文件直接按字节拆分
            const start = chunkIndex * chunkSize;
            const end = Math.min(start + chunkSize, file.size);
            chunk = file.slice(start, end);
        }
        
        // 构造表单数据
        const formData = new FormData();
        formData.append('cvr', cvr);
        formData.append('file', chunk, `${originalFilename}.part${chunkIndex}`);
        formData.append('chunk_index', chunkIndex);
        formData.append('total_chunks', totalChunks);
        formData.append('original_filename', originalFilename);
        
        try {
            const response = await fetch('https://my_url/upload', {
                method: 'POST',
                body: formData
            });
            
            const result = await response.text();
            console.log(`分片 ${chunkIndex+1}/${totalChunks}:`, result);
            
            if (!response.ok) {
                alert(`分片 ${chunkIndex+1} 上传失败`);
                return;
            }
        } catch (error) {
            alert(`分片 ${chunkIndex+1} 上传出错: ${error.message}`);
            return;
        }
    }
    
    alert('所有分片上传完成，服务器正在合并文件');
});
</script>

1. CSV文件处理：按行拆分确保每个分片都是完整的行，避免合并后出现数据错误；首分片保留表头，后续分片仅包含数据行。
2. 二进制文件处理：XLS/XLSX是二进制格式，直接按字节拆分后合并即可恢复原文件，无需额外处理。
3. 临时存储：后端为每个上传创建唯一临时目录，避免并发上传冲突；合并完成后自动清理临时文件。
4. 错误处理：前后端均添加了错误捕获逻辑，便于排查上传失败问题。

内容的提问来源于stack exchange，提问作者Marc