Django Rest大文件后台异步上传超时问题解决方案咨询

核心思路 #

跳过请求处理阶段的本地磁盘写入操作，直接将文件流信息传递给后台任务，或结合Django存储机制实现异步处理，同时保证Celery任务参数可序列化，彻底消除请求超时问题。

方案一：内存直传（推荐） #

利用Django内存存储特性，将文件内容编码后直接传递给Celery任务，完全避免本地磁盘IO。

1. 调整Django上传配置 #

在settings.py中修改文件存储阈值，确保10MB以内的文件全部存在内存中：

# 设置内存存储上限为10MB（匹配你的文件大小限制）
FILE_UPLOAD_MAX_MEMORY_SIZE = 10 * 1024 * 1024
# 禁用临时文件目录（可选，强制符合大小的文件走内存）
FILE_UPLOAD_TEMP_DIR = None

2. 修改Celery任务逻辑 #

直接解码base64格式的文件内容并上传至S3：

from celery import shared_task
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from .models import Property, PropertyPhoto
import base64

@shared_task(bind=True, retry_backoff=3)
def process_property_photos(self, property_id, photo_data_list):
    try:
        property_instance = Property.objects.get(id=property_id)
        for photo_data in photo_data_list:
            # 解码base64文件内容
            file_content = ContentFile(base64.b64decode(photo_data['file_content']), name=photo_data['file_name'])
            # 上传至S3（需提前配置S3为默认存储）
            s3_path = f"property_photos/{property_id}/{photo_data['file_name']}"
            file_url = default_storage.save(s3_path, file_content)
            
            # 创建房源照片记录
            PropertyPhoto.objects.create(
                property=property_instance,
                photo=file_url,
                category_id=photo_data['category_id'],
                content_type=photo_data['content_type'],
                size=photo_data['size']
            )
    except Property.DoesNotExist:
        self.retry(exc=Exception(f"Property {property_id} not found"), max_retries=3)
    except Exception as e:
        self.retry(exc=e, max_retries=3)

3. 重构perform_create方法 #

移除本地写盘逻辑，直接将文件内容编码为base64：

def perform_create(self, serializer):
    from .tasks import process_property_photos
    import base64
    request = self.request
    user = request.user

    try:
        developer_profile = Developer.objects.get(user=user)
    except Developer.DoesNotExist:
        logger.warning(f"User {user.pk} ({user.email}) attempted property creation without a Developer profile.")
        raise serializers.ValidationError(
            {"developer": "Authenticated user does not have an associated developer profile."},
            code=status.HTTP_400_BAD_REQUEST
        )

    amenity_instances = serializer.validated_data.pop('amenity_ids', [])

    photo_data_list = []
    index = 0

    for index in range(15):  
        file_key = f'photos[{index}]file'
        category_key = f'photos[{index}]category_id'
        
        uploaded_file = request.FILES.get(file_key)
        category_id = request.POST.get(category_key)
        
        if uploaded_file:
            MAX_UPLOAD_SIZE = 10 * 1024 * 1024  # 10MB
            if uploaded_file.size > MAX_UPLOAD_SIZE:
                raise serializers.ValidationError({"photos": f"File {uploaded_file.name} exceeds the 10MB limit."})
            
            # 读取文件内容并转base64
            file_content = base64.b64encode(uploaded_file.read()).decode('utf-8')
            
            # 准备可序列化的任务参数
            photo_data = {
                'file_content': file_content,
                'file_name': uploaded_file.name,
                'category_id': int(category_id) if category_id else None,
                'content_type': uploaded_file.content_type,
                'size': uploaded_file.size
            }
            photo_data_list.append(photo_data)
        else:
            break
    logger.info(f"Prepared {len(photo_data_list)} photo(s) for async processing by user {user.pk}.")

    try:
        with transaction.atomic():
            validated_data = serializer.validated_data
            validated_data['listed_at'] = timezone.now()
            property_instance = serializer.save(developer=developer_profile, **validated_data)
            logger.info(f"Property instance {property_instance.id} created for developer {developer_profile.id}.")

            if amenity_instances:
                property_instance.amenities.set(amenity_instances)
                logger.info(f"Set {len(amenity_instances)} amenities for property {property_instance.id}.")

    except Exception as e:
        logger.error(f"Error during property creation (User: {user.id}): {e}", exc_info=True)
        raise serializers.ValidationError(
            {"detail": "An error occurred while saving property details. Please try again."},
            code=status.HTTP_500_INTERNAL_SERVER_ERROR
        )

    try:
        if photo_data_list:
            photo_task = process_property_photos.delay(property_instance.id, photo_data_list)
            logger.info(f"Queued photo processing task {photo_task.id} for property {property_instance.id}")

    except Exception as e:
        logger.error(f"Error queuing background tasks for property {property_instance.id}: {e}", exc_info=True)

    serializer.instance = property_instance

方案二：临时文件优化（兼容大文件场景） #

若担心base64编码内存占用，可保留临时文件逻辑，但优化存储介质并自动清理：

1. 配置快速临时存储 #

在settings.py中将临时目录指向内存文件系统（如tmpfs）：

FILE_UPLOAD_TEMP_DIR = '/dev/shm/django_uploads'  # Linux tmpfs路径，读写速度接近内存

2. 修改Celery任务添加清理逻辑 #

处理完文件后自动删除临时文件：

from celery import shared_task
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from .models import Property, PropertyPhoto
import os

@shared_task(bind=True, retry_backoff=3)
def process_property_photos(self, property_id, photo_data_list):
    try:
        property_instance = Property.objects.get(id=property_id)
        for photo_data in photo_data_list:
            file_path = photo_data['file_path']
            # 读取临时文件并上传S3
            with open(file_path, 'rb') as f:
                file_content = ContentFile(f.read(), name=photo_data['file_name'])
                s3_path = f"property_photos/{property_id}/{photo_data['file_name']}"
                file_url = default_storage.save(s3_path, file_content)
            
            # 创建照片记录
            PropertyPhoto.objects.create(
                property=property_instance,
                photo=file_url,
                category_id=photo_data['category_id'],
                content_type=photo_data['content_type'],
                size=photo_data['size']
            )
            # 清理临时文件
            if os.path.exists(file_path):
                os.remove(file_path)
    except Property.DoesNotExist:
        self.retry(exc=Exception(f"Property {property_id} not found"), max_retries=3)
    except Exception as e:
        self.retry(exc=e, max_retries=3)

额外优化建议 #

• 配置Celery使用Redis/RabbitMQ作为消息队列，确保任务快速入队不阻塞请求
• 启用Celery任务重试机制，处理S3上传网络波动等异常情况
• 前端添加任务状态轮询或WebSocket推送，告知用户上传进度
• 使用Celery Flower监控任务执行状态，快速排查失败任务

内容的提问来源于stack exchange，提问作者Kyamasam