Scrapy为每个Request配置不同Feeds导出路径的问题排查

你的代码存在两个核心错误：

• 类定义阶段无法访问实例变量：custom_settings是类属性，在类被定义时还没有创建任何Spider实例，self此时根本不存在，所以self.feeds_csv_path会直接触发NameError。
• FEEDS配置无法动态更新：即便绕开第一个问题，Scrapy在启动Spider时就会读取并固定FEEDS的配置，后续修改实例属性self.feeds_csv_path不会让FEEDS自动切换输出路径——FEEDS是全局生效的，不能为每个请求单独配置。

要实现每个房产对应一个独立CSV文件，推荐两种可行方式：

方案1：在parse方法中直接写入文件 #

这种方式简单直观，直接在解析每个房产数据时写入对应CSV：

import os
import json
import scrapy
import pandas as pd
from configparser import ConfigParser

class GetpropertyimgurlsSpider(scrapy.Spider):
    name = 'GetPropertyImgUrls'

    def __init__(self, *args, **kwargs):
        super(GetpropertyimgurlsSpider, self).__init__(*args, **kwargs)

    def start_requests(self):
        files = self.get_html_files()
        for file in files[:2]:
            # 将文件路径通过meta传递给parse方法
            yield scrapy.Request(file, callback=self.parse, meta={"html_file_path": file})

    def parse(self, response):
        html_file_path = response.meta["html_file_path"]
        csv_path = self.get_feeds_csv_path(html_file_path)
        # 确保目标目录存在
        os.makedirs(os.path.dirname(csv_path), exist_ok=True)

        texts = response.xpath("//text()").getall()
        json_text = json.loads(texts[1])
        photos = json_text["@graph"][3]["photo"]
        
        # 收集所有图片URL并写入CSV
        img_urls = [photo["contentUrl"] for photo in photos]
        pd.DataFrame({"image_url": img_urls}).to_csv(csv_path, index=False)

    def get_feeds_csv_path(self, html_file_path):
        # 转换file:/// URL为本地路径
        local_path = html_file_path.replace("file:///", "")
        # 从目录名提取房产ID
        property_dir = os.path.basename(os.path.dirname(local_path))
        property_id = property_dir.split("_")[1]
        # 生成正确的CSV路径
        return f"{os.path.dirname(local_path)}/images/Property_{property_id}_ImgSrcs.csv"

    def get_path(self):
        config = ConfigParser()
        config.read("config.ini")
        return config["scrapezoopla"]["path"]

    def get_html_files(self):
        path = self.get_path()
        property_dir = f"{path}/data/properties/"
        folders = [f for f in os.listdir(property_dir) if os.path.isdir(os.path.join(property_dir, f))]
        
        html_files = []
        for folder in folders:
            html_path = os.path.join(property_dir, folder, f"{folder}.html")
            if os.path.isfile(html_path):
                html_files.append(f"file:///{html_path}")
        return html_files

方案2：使用Item Pipeline动态路由输出文件 #

如果想遵循Scrapy的Item流程，可通过Pipeline根据房产ID将数据写入不同文件：

1. 定义Item（items.py） #

import scrapy

class PropertyImageUrlItem(scrapy.Item):
    image_url = scrapy.Field()
    property_id = scrapy.Field()

2. 修改Spider #

import os
import json
import scrapy
from configparser import ConfigParser
from your_project.items import PropertyImageUrlItem  # 替换为你的项目名

class GetpropertyimgurlsSpider(scrapy.Spider):
    name = 'GetPropertyImgUrls'

    def __init__(self, *args, **kwargs):
        super(GetpropertyimgurlsSpider, self).__init__(*args, **kwargs)

    def start_requests(self):
        files = self.get_html_files()
        for file in files[:2]:
            local_path = file.replace("file:///", "")
            property_id = os.path.basename(os.path.dirname(local_path)).split("_")[1]
            yield scrapy.Request(file, callback=self.parse, meta={"property_id": property_id})

    def parse(self, response):
        property_id = response.meta["property_id"]
        texts = response.xpath("//text()").getall()
        json_text = json.loads(texts[1])
        photos = json_text["@graph"][3]["photo"]
        
        for photo in photos:
            item = PropertyImageUrlItem()
            item["image_url"] = photo["contentUrl"]
            item["property_id"] = property_id
            yield item

    # 保留get_path、get_html_files方法，代码同方案1

3. 定义Pipeline（pipelines.py） #

import os
import csv
from configparser import ConfigParser
from your_project.items import PropertyImageUrlItem  # 替换为你的项目名

class PropertyImagePipeline:
    def __init__(self, root_path):
        self.root_path = root_path
        self.writers = {}  # 缓存每个房产ID对应的CSV写入器

    @classmethod
    def from_crawler(cls, crawler):
        config = ConfigParser()
        config.read("config.ini")
        return cls(config["scrapezoopla"]["path"])

    def process_item(self, item, spider):
        if isinstance(item, PropertyImageUrlItem):
            property_id = item["property_id"]
            csv_path = f"{self.root_path}/data/properties/property_{property_id}/images/Property_{property_id}_ImgSrcs.csv"
            os.makedirs(os.path.dirname(csv_path), exist_ok=True)
            
            # 初始化对应房产的CSV写入器
            if property_id not in self.writers:
                file = open(csv_path, "w", newline="", encoding="utf-8")
                writer = csv.DictWriter(file, fieldnames=["image_url"])
                writer.writeheader()
                self.writers[property_id] = writer
            
            self.writers[property_id].writerow({"image_url": item["image_url"]})
            return item

    def close_spider(self, spider):
        # 关闭所有打开的文件句柄
        for writer in self.writers.values():
            writer.file.close()

4. 启用Pipeline（settings.py） #

ITEM_PIPELINES = {
    'your_project.pipelines.PropertyImagePipeline': 300,
}

• 原代码中get_feeds_csv_path的路径逻辑错误：需要先将file:///开头的URL转换为本地路径，且不能把HTML文件路径当作目录，应取其父目录作为基准。
• 方案1适合小规模数据，实现简单；方案2更符合Scrapy架构，适合复杂爬虫流程。

内容的提问来源于stack exchange，提问作者Andrea