Scrapy爬虫图片下载MD5重命名及JSON字段添加问题求助

你的爬虫图片下载失败的核心原因是Pipeline的file_path方法中错误地将列表类型的item["path"]调用split方法，导致抛出AttributeError中断下载。同时，提前在爬虫中生成path列表的方式无法匹配后续追加的图片URL，会出现路径对应错误。

以下是修正后的完整实现，满足图片MD5重命名、层级存储，以及将MD5路径写入JSON的需求：

1. 修改爬虫代码（移除冗余的路径生成逻辑） #

删除爬虫中的make_path函数，以及所有path字段的赋值：

import scrapy
import json


class HouzzSimilar(scrapy.Spider):
    name = "houzz_crawler"

    custom_settings = {
        "LOG_FILE": "houzz_spider.log",
        "IMAGES_STORE": "houzz_images",
        "FEEDS": {
            "houzz.json": {
                "format": "json",
            }
        },
        "ITEM_PIPELINES": {
            "houzz_crawler.pipelines.HouzzImagePipeline": 1,
        },
    }

    # 保留原headers、cookies、base_url、similar_ideas_api_url定义

    def start_requests(self):
        yield scrapy.Request(
            url=self.base_url, headers=self.headers, callback=self.parse_ideas
        )

    def parse_ideas(self, response):
        ideas = response.css("a.hz-photo-card__ratio-box::attr(href)").extract()
        for idea in ideas:
            yield scrapy.Request(
                url=idea, headers=self.headers, callback=self.parse_project_url
            )

    def parse_project_url(self, response):
        data = response.css('script[id="hz-ctx"] ::text').get()
        json_data = json.loads(data)
        space_id = json_data["data"]["pageContentData"]["spaceId"]
        space = json_data["data"]["stores"]["data"]["SpaceStore"]["data"][space_id]
        project_id = space["projectId"]
        space_url = space["url"]
        raw_project_url = (
            space_url.split("~")[0].replace("phvw", "pj").replace("vp", "vj")
        )
        project_url = raw_project_url + "~" + str(project_id)
        yield scrapy.Request(
            url=project_url, headers=self.headers, callback=self.parse_project_idea
        )

    def parse_project_idea(self, response):
        idea_board = response.css(
            "div.hz-prj-container.hz-prj-container__photos.clearfix ::attr(href)"
        ).extract()
        for idea_link in idea_board:
            yield scrapy.Request(
                url=idea_link,
                headers=self.headers,
                callback=self.parse_idea_details,
            )

    def parse_idea_details(self, response):
        item = {}
        item["ideadId"] = response.url.split("~")[-1]
        item["ideaUrl"] = response.url
        item["Title"] = response.css(
            "h1.hz-view-photo__space-info__title.text-bold::text"
        ).get()

        item["imageURL"] = response.css(
            "div.view-photo-image-pane > img::attr(src)"
        ).extract()
        item["image_urls"] = item["imageURL"].copy()
        item["similarIdeas"] = []

        spaceId = response.url.split("~")[-1]
        body = f"spaceId={spaceId}&fromItem=0&itemsPerPage=10&contentDescriptor=%7B%22t%22%3A1%2C%22et%22%3A3%2C%22id%22%3A160668148%7D"
        yield scrapy.Request(
            url=self.similar_ideas_api_url,
            method="POST",
            cookies=self.cookies,
            headers=self.headers,
            body=body,
            cb_kwargs={"item": item},
            callback=self.get_similar_ideas_urls,
        )

    def get_similar_ideas_urls(self, response, item=None):
        data = response.json()["spaceData"]["spaces"]
        space_keys = list(data.keys())
        space_urls = set([data[key]["url"] for key in space_keys])
        yield scrapy.Request(
            url=space_urls.pop(),
            headers=self.headers,
            cb_kwargs={"item": item, "space_urls": space_urls},
            callback=self.parse_similar_ideas,
        )

    def parse_similar_ideas(self, response, item=None, space_urls=None):
        image_urls = response.css(
            "div.view-photo-image-pane > img::attr(src)"
        ).extract()
        item["image_urls"] += image_urls
        item["similarIdeas"].append(
            {
                "ideaId": response.url.split("~")[-1],
                "ideaUrl": response.url,
                "Title": response.css(
                    "h1.hz-view-photo__space-info__title.text-bold::text"
                ).get(),
                "image_urls": image_urls,
            }
        )
        if len(space_urls) > 0:
            yield scrapy.Request(
                url=space_urls.pop(),
                headers=self.headers,
                cb_kwargs={"item": item, "space_urls": space_urls},
                dont_filter=True,
                callback=self.parse_similar_ideas,
            )
        else:
            yield item

2. 修正ImagePipeline代码 #

在Pipeline中统一处理MD5生成、路径构建，以及将下载路径映射到item字段：

import hashlib
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem


class HouzzImagePipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        # 为每个图片请求添加元信息，标记所属类型和索引
        # 处理主item的图片
        for idx, url in enumerate(item["image_urls"]):
            yield scrapy.Request(url, meta={"item_type": "main", "index": idx})
        # 处理similarIdeas中的图片
        for idea_idx, idea in enumerate(item["similarIdeas"]):
            for img_idx, url in enumerate(idea["image_urls"]):
                yield scrapy.Request(url, meta={"item_type": "similar", "idea_idx": idea_idx, "img_idx": img_idx})

    def file_path(self, request, response=None, info=None, *, item=None):
        # 根据图片URL生成MD5哈希
        url_hash = hashlib.md5(request.url.encode('utf-8')).hexdigest()
        # 构建层级存储路径：前3位/中间3位/后3位/完整哈希.jpg
        return f"{url_hash[:3]}/{url_hash[3:6]}/{url_hash[6:9]}/{url_hash}.jpg"

    def item_completed(self, results, item, info):
        # 初始化存储路径的字段
        item["image_paths"] = []
        for idea in item["similarIdeas"]:
            idea["image_paths"] = []

        for success, data in results:
            if not success:
                raise DropItem(f"图片下载失败: {data.get('url', '未知URL')}")
            else:
                request = data['request']
                file_path = data['path']
                meta = request.meta

                # 根据元信息将路径对应到item的正确位置
                if meta["item_type"] == "main":
                    idx = meta["index"]
                    if idx < len(item["image_paths"]):
                        item["image_paths"][idx] = file_path
                    else:
                        item["image_paths"].append(file_path)
                elif meta["item_type"] == "similar":
                    idea_idx = meta["idea_idx"]
                    img_idx = meta["img_idx"]
                    idea = item["similarIdeas"][idea_idx]
                    if img_idx < len(idea["image_paths"]):
                        idea["image_paths"][img_idx] = file_path
                    else:
                        idea["image_paths"].append(file_path)

        # 清理冗余字段
        if "imageURL" in item:
            del item["imageURL"]

        return item

3. 实现效果说明 #

• 图片存储：图片会被下载到houzz_images目录下的层级文件夹中，格式为c69/96d/f0d/c6996df0d9d852f1f39fcb7074ace625.jpg。
• JSON输出：最终的houzz.json中，每个item会新增image_paths字段（存储主图片的MD5路径），每个similarIdeas对象会新增image_paths字段（存储对应子图片的MD5路径）。

内容的提问来源于stack exchange，提问作者X-something