使用Scrapy爬取Pararius.nl遇Fairlane保护，求绕过方案及示例

核心结论 #

肯定需要借助工具，Fairlane这类反爬会检测请求特征、JS渲染能力、访问频率，仅靠静态Scrapy请求很难绕过。推荐两种方案：优化Scrapy请求特征+Cookie池（基础版），或者Scrapy集成Playwright渲染（进阶版，成功率更高）。

方案一：基础版（纯Scrapy优化） #

通过模拟真实浏览器请求特征、控制访问频率来降低被拦截概率，适合简单场景。

1. 调整Scrapy配置（settings.py） #

# 禁用默认Cookie中间件，手动管理Cookie
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': None,
}

# 设置请求延迟，模拟真实用户访问间隔
DOWNLOAD_DELAY = 3
# 随机化延迟范围
RANDOMIZE_DOWNLOAD_DELAY = True

# 模拟真实浏览器的请求头（从Chrome/Firefox开发者工具复制）
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Referer': 'https://www.pararius.nl/',
    'Connection': 'keep-alive',
}

2. 优化爬虫代码 #

手动从浏览器获取有效Cookie（访问Pararius首页，复制开发者工具中的Cookie），并调整请求逻辑：

import scrapy

class ParariusSpider(scrapy.Spider):
    name = 'pararius'
    allowed_domains = ['pararius.nl']
    
    # 手动从浏览器复制的有效Cookie
    cookies = {
        'your_cookie_key1': 'your_cookie_value1',
        'your_cookie_key2': 'your_cookie_value2',
        # 补充完整Cookie内容
    }

    def start_requests(self):
        # 先请求首页，建立会话
        yield scrapy.Request(
            url='https://www.pararius.nl/',
            callback=self.parse_home,
            cookies=self.cookies,
            headers=self.settings.get('DEFAULT_REQUEST_HEADERS')
        )

    def parse_home(self, response):
        url_template = 'https://www.pararius.nl/{deal_type}/nederland/p-{page}/'
        for deal_type in ['huurwoningen', 'koopwoningen']:
            for page in range(1, 2):
                yield scrapy.Request(
                    url=url_template.format(deal_type=deal_type, page=page),
                    callback=self.parse_pages,
                    cookies=self.cookies,
                    headers=self.settings.get('DEFAULT_REQUEST_HEADERS'),
                    cb_kwargs={'deal_type': deal_type}
                )

    def parse_pages(self, response, deal_type):
        self.logger.info(f"成功访问: {response.url}")
        # 这里可以继续解析页面内容
        return

方案二：进阶版（Scrapy + Playwright） #

Fairlane会验证JS执行环境，Playwright可以模拟真实浏览器的渲染行为，几乎完全模拟用户操作，拦截概率极低。

1. 安装依赖 #

pip install scrapy-playwright playwright
playwright install chromium

2. 配置settings.py #

# 启用Playwright中间件
DOWNLOADER_MIDDLEWARES = {
    'scrapy_playwright.middleware.PlaywrightMiddleware': 543,
}

# 配置Playwright
PLAYWRIGHT_LAUNCH_OPTIONS = {
    'headless': False,  # 调试时可以设为False，观察浏览器行为
    'args': ['--start-maximized', '--disable-blink-features=AutomationControlled'],
}

# 禁用默认下载器，使用Playwright下载器
DOWNLOAD_HANDLERS = {
    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}

3. 爬虫代码示例 #

import scrapy
from scrapy_playwright.page import PageCoroutine

class ParariusPlaywrightSpider(scrapy.Spider):
    name = 'pararius_playwright'
    allowed_domains = ['pararius.nl']

    def start_requests(self):
        # 先访问首页，模拟用户浏览路径
        yield scrapy.Request(
            url='https://www.pararius.nl/',
            callback=self.parse_home,
            meta={
                'playwright': True,
                'playwright_page_coroutines': [
                    PageCoroutine('wait_for_selector', 'div.listings-container'),  # 等待页面加载完成
                ],
            }
        )

    def parse_home(self, response):
        url_template = 'https://www.pararius.nl/{deal_type}/nederland/p-{page}/'
        for deal_type in ['huurwoningen', 'koopwoningen']:
            for page in range(1, 2):
                yield scrapy.Request(
                    url=url_template.format(deal_type=deal_type, page=page),
                    callback=self.parse_pages,
                    meta={
                        'playwright': True,
                        'playwright_page_coroutines': [
                            PageCoroutine('wait_for_selector', 'div.listing-item'),  # 等待列表项加载
                            PageCoroutine('scroll_by', 0, 500),  # 模拟滚动，触发JS加载
                        ],
                    },
                    cb_kwargs={'deal_type': deal_type}
                )

    def parse_pages(self, response, deal_type):
        self.logger.info(f"成功访问: {response.url}")
        # 解析页面内容，比如提取房源链接
        listings = response.css('div.listing-item a::attr(href)').getall()
        for listing in listings:
            yield response.follow(listing, callback=self.parse_listing, meta={'playwright': True})

    def parse_listing(self, response):
        # 解析房源详情页
        title = response.css('h1::text').get()
        price = response.css('span.price::text').get()
        yield {
            'title': title,
            'price': price,
            'url': response.url
        }

注意事项 #

• 实操练习时控制请求量，避免频繁访问给网站造成压力。
• 定期更换User-Agent，或者使用User-Agent池。
• Playwright的headless模式设为True可提升效率，添加--disable-blink-features=AutomationControlled参数可绕过自动化检测。

内容的提问来源于stack exchange，提问作者parastoo