Flask集成Scrapy爬虫未加载自定义settings致403请求失败

我有一个可正常运行的Scrapy爬虫，集成到Flask Web应用后，爬虫能按需求启动，但不会加载settings.py文件中的自定义配置，因此返回403响应。注意通过命令行运行爬虫时，配置可正常生效，请问如何让Flask应用运行爬虫时加载正确的配置？

爬虫代码 #

class PostsSpider(scrapy.Spider):

    page_number = 0
    name = 'posts'
    allowed_domains = ['ra.co']

    # global artist
    # artist = input('Artist name:')
    start_urls = [
        f'https://ra.co/dj/ouladomar/past-events?'
    ]

    def parse(self, response):
        if len(response.css('li.Column-sc-18hsrnn-0.inVJeD')) == 0:
            raise CloseSpider('No more events')

        for link in response.css('li.Column-sc-18hsrnn-0.inVJeD div h3 a::attr(href)'):
            yield response.follow(link.get(), callback=self.parse_act)

        next_page = f'https://ra.co/dj/ouladomar/past-events?page={str(PostsSpider.page_number)}'

        PostsSpider.page_number += 1
        print(PostsSpider.page_number)
        yield response.follow(next_page, callback=self.parse)

    def parse_act(self, response):
        date = response.xpath('//*[@id="__next"]/div[2]/header/div/div[2]//div/ul/li[2]/div/div[2]/a/span/text()').get()
        event = response.xpath('//*[@id="__next"]/div[2]/header/div//div/div/div[2]/h1/span/text()').get()
        promotors = response.xpath('//*[@id="__next"]/div[2]/header/div/div[2]/div[2]/div/ul/li[3]/div/div[2]/a/span/text()').getall()
        location = response.xpath('//*[@id="__next"]/div[2]/header/div//div[1]/div/div/div[1]/nav/ul/li[1]/div/a/span/text()').get()
        country = response.xpath('//*[@id="__next"]/div[2]/header/div//div[1]/div/div/div[1]/nav/ul/li[1]/div/a').attrib['href']
        venue = response.xpath('//*[@id="__next"]/div[2]/header/div/div[2]//div/ul/li[1]/div//span/text()')[1].get()
        acts = response.xpath('//*[@id="__next"]/div[2]/section[1]/div/section[1]/div/div/div[2]/ul/li[1]/div/span/a/span/text()').getall()

        date = re.sub(r'^.*?, ', '', date)

        promotors = ', '.join(promotors)

        if len(date) == 4:
            date = f'99-99-{date}'

        elif len(date) >= 15:
            date = date[5:]

        elif date[-4: -2] == '20':
            date = datetime.strptime(date, '%b %d, %Y').strftime('%d-%m-%Y')
        else:
            date = datetime.strptime(date, '%d %b').strftime('%d-%m') + '-2023'

        country = country.split('/')[-2].upper()

        acts = ', '.join(acts)

        item = {
            'date': date,
            'Event': event,
            'promotors': promotors,
            'Location': location,
            'Country': country,
            'Venue': venue,
            'Acts': acts
        }

        yield item

if __name__== '__main__':
    process = CrawlerProcess(get_project_settings())
    process.crawl(PostsSpider)
    process.start()

Flask应用代码 #

import subprocess

import crochet
crochet.setup()

from flask import Flask , render_template, jsonify, request, redirect, url_for
from scrapy import signals
from scrapy.crawler import CrawlerRunner
from scrapy.signalmanager import dispatcher
import time
    
from ScrapyProject.scraperra.scraperra.spiders.artistspider import PostsSpider

# Creating Flask App Variable

app = Flask(__name__)

output_data = []
crawl_runner = CrawlerRunner()


# By Deafult Flask will come into this when we run the file
@app.route('/')
def index():
    return render_template("index.html")  # Returns index.html file in templates folder.


# After clicking the Submit Button FLASK will come into this
@app.route('/', methods=['POST'])
def submit():
    if request.method == 'POST':
        return redirect(url_for('scrape'))  # Passing to the Scrape function


@app.route("/scrape")
def scrape():
    process = subprocess.Popen('python ScrapyProject/scraperra/scraperra/spiders/artistspider.py', shell=True)
    process.wait()
    return jsonify(output_data)  # Returns the scraped data after being running for 20 seconds.


if __name__ == "__main__":
    app.run(debug=True)

问题核心是用subprocess直接运行爬虫文件时，Scrapy无法识别项目根目录，导致加载不到settings.py。以下是两种可行的解决办法：

方案一：修正subprocess命令，调用Scrapy官方启动方式 #

直接运行爬虫文件不会自动加载项目配置，改用scrapy crawl命令启动，它会自动识别项目根目录下的settings.py：

@app.route("/scrape")
def scrape():
    # 切换到Scrapy项目根目录（即scrapy.cfg所在的路径）
    process = subprocess.Popen(
        'scrapy crawl posts', 
        cwd='ScrapyProject/scraperra/',
        shell=True
    )
    process.wait()
    
    # 读取爬虫输出结果（需提前在settings.py中配置FEEDS输出到文件）
    import json
    with open('items.json', 'r') as f:
        output_data = json.load(f)
    return jsonify(output_data)

方案二：用CrawlerRunner加载项目配置（推荐） #

利用Scrapy的get_project_settings()加载配置，结合Crochet实现异步爬虫，无需依赖subprocess，还能直接收集爬虫数据：

import crochet
crochet.setup()

from flask import Flask, render_template, jsonify, request, redirect, url_for
from scrapy import signals
from scrapy.crawler import CrawlerRunner
from scrapy.signalmanager import dispatcher
from scrapy.utils.project import get_project_settings
import os
import time

# 指定Scrapy配置模块路径
os.environ['SCRAPY_SETTINGS_MODULE'] = 'scraperra.settings'

from ScrapyProject.scraperra.scraperra.spiders.artistspider import PostsSpider

app = Flask(__name__)
output_data = []
# 用项目配置初始化CrawlerRunner
crawl_runner = CrawlerRunner(get_project_settings())

# 监听爬虫信号，收集爬取到的数据
def collect_items(item):
    output_data.append(item)

dispatcher.connect(collect_items, signal=signals.item_scraped)

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/', methods=['POST'])
def submit():
    if request.method == 'POST':
        return redirect(url_for('scrape'))

@app.route("/scrape")
def scrape():
    global output_data
    output_data = []  # 清空历史数据
    # 异步启动爬虫
    crawl_runner.crawl(PostsSpider)
    # 根据爬虫实际耗时调整等待时间
    time.sleep(10)
    return jsonify(output_data)

if __name__ == "__main__":
    app.run(debug=True)

关键说明 #

• 直接运行python xxx.py启动爬虫时，Scrapy不会自动加载项目配置，必须显式调用get_project_settings()；而scrapy crawl命令会自动识别项目根目录的配置文件。
• 方案二无需依赖外部进程，能直接通过信号收集数据，是更优雅的集成方式。

内容的提问来源于stack exchange，提问作者AxelotlZ