Scrapy ImagesPipeline丢弃图片仍被导出问题求助

我使用Scrapy的ImagesPipeline根据校验和过滤下载图片，日志显示已丢弃598张、爬取2张，但文件夹中所有600张图片均被保存。试过相关解决方案但无效，以下是我的代码及运行日志：

pipelines.py #

from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request
from scrapy.exceptions import DropItem


class NikeNamePipeline(ImagesPipeline):


    def get_media_requests(self, item, info):

        urls = ItemAdapter(item).get(self.images_urls_field, [])
        return [Request(u, meta={'imagename': item.get('image_name')}) for u in urls]

        
    def file_path(self, request, response=None, info=None, *, item=None):
                    
        return f"full/{request.meta['imagename']}.jpg"


    def thumb_path(self,request,thumb_id,response=None, *, item=None, info=None):

        return f"thumbs/small/{request.meta['imagename']}.jpg"
    

    def item_completed(self, results, item, info):

        checksum = results[0][1].get('checksum')
        
        if checksum == 'e00d8ea516c4246c25930163241ec745':
            
            raise DropItem('do not download')

settings.py #

ITEM_PIPELINES = {
    'nike_name.pipelines.NikeNamePipeline': 1,
}


IMAGES_STORE = 'C://Users/USER/scrape_projects/nike_name'

DOWNLOAD_TIMEOUT = 1200

IMAGES_THUMBS = {
    'small': (50, 50),
    }

运行日志 #

{'downloader/request_bytes': 435600,
'downloader/request_count': 1200,
'downloader/request_method_count/GET': 1200,
'downloader/response_bytes': 14493780,
'downloader/response_count': 1200,
'downloader/response_status_count/200': 1200,
'elapsed_time_seconds': 23.401415,
'file_count': 600,
'file_status_count/downloaded': 600,
'finish_reason': 'finished',
'finish_time': datetime.datetime(2022, 8, 22, 11, 15, 42, 53919),
'item_dropped_count': 598,
'item_dropped_reasons_count/DropItem': 598,
'item_scraped_count': 2,
'log_count/DEBUG': 1802,
'log_count/INFO': 10,
'log_count/WARNING': 598,
'response_received_count': 1200,
'scheduler/dequeued': 600,
'scheduler/dequeued/memory': 600,
'scheduler/enqueued': 600,
'scheduler/enqueued/memory': 600,

问题核心：item_completed方法是所有图片下载完成后才触发，此时文件已经写入磁盘，抛出DropItem只能丢弃Item对象，无法删除已保存的图片。要实现过滤，必须在文件保存前做校验判断。

修改后的pipelines.py代码：

from itemadapter import ItemAdapter
from scrapy.pipelines.images import ImagesPipeline
from scrapy import Request
from scrapy.exceptions import DropItem


class NikeNamePipeline(ImagesPipeline):

    def get_media_requests(self, item, info):
        urls = ItemAdapter(item).get(self.images_urls_field, [])
        return [Request(u, meta={'imagename': item.get('image_name')}) for u in urls]

    def file_path(self, request, response=None, info=None, *, item=None):
        return f"full/{request.meta['imagename']}.jpg"

    def thumb_path(self, request, thumb_id, response=None, *, item=None, info=None):
        return f"thumbs/small/{request.meta['imagename']}.jpg"

    def persist_file(self, path, image, buf, info, *, item=None):
        # 计算文件校验和，和item_completed中获取的校验和逻辑一致
        checksum = self.file_checksum(buf)
        if checksum == 'e00d8ea516c4246c25930163241ec745':
            # 跳过保存操作
            return
        # 符合条件则调用父类方法保存文件
        super().persist_file(path, image, buf, info, item=item)

    def item_completed(self, results, item, info):
        # 可选：处理下载失败的情况
        for ok, data in results:
            if not ok:
                raise DropItem(f"图片下载失败: {data}")
        return item

说明 #

1. persist_file是ImagesPipeline负责将文件写入磁盘的核心方法，在这个阶段判断校验和，直接跳过不符合条件的文件保存。
2. file_checksum是父类FilesPipeline的内置方法，计算的MD5校验和与item_completed中获取的完全一致。
3. 修改后，不符合校验和的图片不会被保存到full和thumbs目录，日志中的file_status_count/downloaded会对应实际保存的数量。

内容的提问来源于stack exchange，提问作者chaim18