Scrapy的FEEDS配置无法保存输出文件的问题求助

我写了一个批量检查网站响应状态的Scrapy爬虫，还用Tkinter做了个简单桌面GUI来运行它。爬虫能正常爬取数据，但FEEDS配置没法把结果保存到文件里。下面是相关代码和运行输出：

爬虫代码 #

import scrapy
import pandas as pd
from twisted.internet.error import DNSLookupError


class CheckSpider(scrapy.Spider):
    name = 'check'

    def read_xl(df):
        df = pd.read_excel('url.xlsx')
        return df['url'].tolist()

    def start_requests(self):
        for value in self.read_xl():
            yield scrapy.Request(
                url=value,
                callback= self.parse,
                errback=self.parse_error
            )
        return super().start_requests()

    def parse_error(self, failure):
        if failure.check(DNSLookupError):
            request = failure.request
            yield {
                'URL': request.url,
                'Status': failure.value
            }

    def parse(self, response):
        if response.request.meta.get('redirect_urls'):
            yield {
                'URL': response.request.meta.get('redirect_urls')[0],
                'Redireted URL': response.request.url,
                'Status': response.status
            }
        else:
            yield {
                'URL': response.url,
                'Redireted URL': response.request.url,
                'Status': response.status
            }

GUI代码（apps.py） #

from tkinter import *
from tkinter import messagebox
from tkinter import filedialog
from scrapy.utils import project
from scrapy import spiderloader
from scrapy.utils.log import configure_logging
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor
import threading


def get_spiders():
    settings = project.get_project_settings()
    spider_loader = spiderloader.SpiderLoader.from_settings(settings)
    return spider_loader.list()

def get_chosen_spider(value):
    global chosen_spider
    chosen_spider = value
    return chosen_spider

def get_chosen_feed(value):
    global chosen_feed
    chosen_feed = value
    return chosen_feed

def browse_button():
    global folder_path
    folder_path = filedialog.askdirectory()
    folder_path_entry.delete(0, END)
    folder_path_entry.insert(0, folder_path)
    return folder_path

def execute_spider():
    if dataset_entry.get() == '' or chosen_feed not in ['CSV', 'JSON']:
        messagebox.showerror('Error', 'All entries are required')
        return
    try:
        feed_uri = f'file:///{folder_path}/{dataset_entry.get()}.{chosen_feed}'
    except:
        messagebox.showerror('Error', 'All entries are required')

    settings = project.get_project_settings()
    # settings.set('FEED_URI', feed_uri)
    # settings.set('FEED_TYPE', chosen_feed)
    settings.set("FEEDS", {
        f'output.{chosen_feed}': {
            'format': chosen_feed,
            'encoding': 'utf8'
        }
    })

    configure_logging()
    runner = CrawlerRunner(settings)
    runner.crawl(chosen_spider)

    reactor.run(installSignalHandlers=False)

def start_execute_thread(event):
    global execute_thread
    execute_thread = threading.Thread(target=execute_spider, daemon=True)
    execute_thread.start()
    app.after(10, check_execute_thread)

def check_execute_thread():
    if execute_thread.is_alive():
        app.after(10, check_execute_thread)


app = Tk()
# app title
app.title('Check Website Status')
# app size
app.geometry('300x200')
app.resizable(False, False)

# app label
spider_label = Label(app, text='Choose a spider')
spider_label.grid(row=0, column=0, sticky=W, pady=10, padx=10)

# Choose Spider
spider_text = StringVar(app)
spider_text.set('Choose a spider')
spiders = [spider for spider in get_spiders()]

spiders_dropdown = OptionMenu(app, spider_text, *spiders, command=get_chosen_spider)
spiders_dropdown.grid(row=0, column=1, columnspan=2)

# Feed Type
feed_label = Label(app, text='Choose a feed')
feed_label.grid(row=1, column=0, sticky=W, pady=10, padx=10)

feed_text = StringVar(app)
feed_text.set('Choose a spider')
feeds = ['CSV', 'JSON']

feed_dropdown = OptionMenu(app, feed_text, *feeds, command=get_chosen_feed)
feed_dropdown.grid(row=1, column=1, columnspan=2)

# path entry
folder_path_text = StringVar(app)
folder_path_entry = Entry(app, textvariable=folder_path_text)
folder_path_entry.grid(row=2, column=0, pady=10, padx=10)

# Dataset entry
dataset_text = StringVar(app)
dataset_entry = Entry(app, textvariable=dataset_text, width=10)
dataset_entry.grid(row=2, column=1, pady=10, padx=10)

browse_btn = Button(app, text='Browse', command=browse_button)
browse_btn.grid(row=2, column=2,)

execute_btn = Button(app,text='Execute', command=lambda: start_execute_thread(None))
execute_btn.grid(row=3, column=0, columnspan=3)

app.mainloop()

运行输出 #

{'URL': 'https://equbot.com/our-customers/', 'Redireted URL': 'https://equbot.com/our-customers/', 'Status': 200}
2022-11-03 00:59:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.fincura.com/> (referer: None)
2022-11-03 00:59:43 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.buxtonco.com/customers>
{'URL': 'https://www.buxtonco.com/clients', 'Redireted URL': 'https://www.buxtonco.com/customers', 'Status': 200}
2022-11-03 00:59:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.features-analytics.com/> (referer: None)
2022-11-03 00:59:43 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.fincura.com/>
{'URL': 'https://www.fincura.com/', 'Redireted URL': 'https://www.fincura.com/', 'Status': 200}
2022-11-03 00:59:43 [scrapy.downloadermiddlewares.redirect] DEBUG: Redirecting (301) to <GET https://www.eventus.com/> from <GET https://www.eventussystems.com/>
2022-11-03 00:59:43 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.features-analytics.com/>
{'URL': 'https://www.features-analytics.com/', 'Redireted URL': 'https://www.features-analytics.com/', 'Status': 200}
2022-11-03 00:59:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://fincad.com/> (referer: None)
2022-11-03 00:59:43 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://www.featurespace.com/customers/> (referer: None)
2022-11-03 00:59:43 [scrapy.core.scraper] DEBUG: Scraped from <200 https://fincad.com/>
{'URL': 'https://fincad.com/', 'Redireted URL': 'https://fincad.com/', 'Status': 200}
2022-11-03 00:59:43 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.featurespace.com/customers/>
{'URL': 'https://www.featurespace.com/customers/', 'Redireted URL': 'https://www.featurespace.com/customers/', 'Status': 200}

问题原因及修复方案 #

核心问题 #

1. FEEDS配置未使用用户选择的路径和文件名，硬编码成了固定的output.{chosen_feed}，完全忽略了GUI中用户输入的路径和文件名。
2. 爬虫的read_xl方法定义错误，类方法第一个参数应为self，否则调用时会触发参数不匹配的错误。
3. FEEDS的format参数需要小写（如csv、json），而代码中传入的是大写的CSV/JSON，不符合Scrapy要求。

修复步骤 #

1. 修正爬虫的read_xl方法：

def read_xl(self):
    df = pd.read_excel('url.xlsx')
    return df['url'].tolist()

2. 修复GUI中FEEDS的配置逻辑，替换execute_spider函数内的settings.set部分：

import os

# 拼接用户选择的完整文件路径
file_path = os.path.join(folder_path, f"{dataset_entry.get()}.{chosen_feed.lower()}")
# 转换为Scrapy兼容的file URI格式（处理Windows路径斜杠问题）
file_uri = f"file:///{file_path.replace(os.sep, '/')}"

settings.set("FEEDS", {
    file_uri: {
        'format': chosen_feed.lower(),
        'encoding': 'utf8',
        'overwrite': True  # 每次运行覆盖旧文件，避免追加内容
    }
})

3. 初始化全局变量，避免未赋值导致的报错，在GUI代码开头添加：

chosen_spider = None
chosen_feed = None
folder_path = None

修复后说明 #

• 现在FEEDS会使用用户选择的文件夹和输入的文件名生成结果文件，路径格式符合Scrapy要求
• 修正了爬虫方法的定义错误，避免潜在崩溃
• 添加overwrite选项，确保每次运行生成全新的结果文件

内容的提问来源于stack exchange，提问作者Raisul Islam