Scrapy爬虫批量爬取子页提取首邮箱故障：仅首个网站有效

需求：遍历目标网站的所有子页，提取第一个出现的邮箱地址，但当前仅第一个网站能成功爬取，后续网站均无法正常工作。

原代码 #

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.crawler import CrawlerProcess
from selenium import webdriver
from selenium.webdriver.common.by import By
import time

class MySpider(CrawlSpider):
    name = 'myspider'
    allowed_domains = []  # 动态设置
    start_urls = []  # 动态设置

    rules = (
        Rule(LinkExtractor(), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        email = response.xpath('//a[contains(@href, "mailto:")][1]/@href').get()
        if email:
            yield {'email': email}
            raise CloseSpider('Email found, spider stopped')


driver = webdriver.Chrome()
driver.get('https://www.houzz.com/professionals/kitchen-and-bath/probr0-bo~t_11790?fi=15')
time.sleep(7)

original_handle = driver.current_window_handle

name = driver.find_elements(By.XPATH, '//a[@class="hz-pro-ctl"]')
address = driver.find_elements(By.XPATH, '//span[@class="hz-pro-search-result__location-info__text"]')
for name, address in zip(name, address):
    name.click()
    time.sleep(5)
    driver.switch_to.window(driver.window_handles[1])
    driver.execute_script("window.scrollTo(0, 2000);")
    time.sleep(10)
    time.sleep(2)
    print('Success')
    time.sleep(10)
    url = driver.find_element(By.XPATH, '//a[@class="sc-62xgu6-0 cZBXc sc-mwxddt-0 kCqoeY hui-link"]')
    url.click()
    time.sleep(10)
    driver.switch_to.window(driver.window_handles[2])
    new_url = driver.current_url
    MySpider.allowed_domains = [new_url.split('/')[2]]
    MySpider.start_urls = [new_url]

    # 启动Scrapy爬虫
    process = CrawlerProcess()
    process.crawl(MySpider)
    process.start()

    driver.close()
    driver.switch_to.window(driver.window_handles[1])
    driver.close()
    driver.switch_to.window(original_handle)

核心问题分析 #

• CrawlerProcess阻塞主线程：process.start()是阻塞调用，第一次启动爬虫后，主线程会卡在这，循环后续的迭代完全无法执行。
• Spider类属性污染：直接修改MySpider.allowed_domains和start_urls属于修改类属性，多次运行会导致状态混乱，且线程不安全。
• CloseSpider未导入：parse_item里抛出CloseSpider但未导入该异常类，会触发NameError，导致爬虫异常退出。
• Selenium窗口操作逻辑脆弱：依赖固定索引切换窗口，关闭窗口后句柄列表长度变化，后续操作会抛出索引越界错误；过度使用time.sleep，页面加载不稳定时会导致元素定位失败。
• 重复创建CrawlerProcess：每次循环新建Process，资源消耗大且不符合Scrapy运行规范。

修复后的代码 #

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import CloseSpider
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from twisted.internet import reactor

class MySpider(CrawlSpider):
    name = 'myspider'
    
    def __init__(self, start_url, allowed_domain, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.start_urls = [start_url]
        self.allowed_domains = [allowed_domain]

    rules = (
        Rule(LinkExtractor(), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        email = response.xpath('//a[contains(@href, "mailto:")][1]/@href').get()
        if email:
            print(f"提取到邮箱: {email}")
            yield {'email': email}
            raise CloseSpider('Email found, spider stopped')

def run_spider(start_url, allowed_domain):
    runner = CrawlerRunner()
    d = runner.crawl(MySpider, start_url=start_url, allowed_domain=allowed_domain)
    d.addBoth(lambda _: reactor.stop())
    reactor.run(installSignalHandlers=False)

# Selenium操作部分
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)

try:
    driver.get('https://www.houzz.com/professionals/kitchen-and-bath/probr0-bo~t_11790?fi=15')
    # 等待目标元素加载完成
    wait.until(EC.presence_of_all_elements_located((By.XPATH, '//a[@class="hz-pro-ctl"]')))
    
    original_handle = driver.current_window_handle
    name_elements = driver.find_elements(By.XPATH, '//a[@class="hz-pro-ctl"]')
    
    for idx, name_elem in enumerate(name_elements):
        try:
            # 重新定位元素，避免 stale element 异常
            name_elem = driver.find_elements(By.XPATH, '//a[@class="hz-pro-ctl"]')[idx]
            name_elem.click()
            
            # 等待新窗口打开并切换
            wait.until(lambda d: len(d.window_handles) > 1)
            driver.switch_to.window(driver.window_handles[1])
            
            # 滚动页面并等待目标链接可点击
            driver.execute_script("window.scrollTo(0, 2000);")
            url_elem = wait.until(EC.element_to_be_clickable((By.XPATH, '//a[@class="sc-62xgu6-0 cZBXc sc-mwxddt-0 kCqoeY hui-link"]')))
            url_elem.click()
            
            # 等待第三个窗口打开并切换
            wait.until(lambda d: len(d.window_handles) > 2)
            driver.switch_to.window(driver.window_handles[2])
            
            new_url = driver.current_url
            allowed_domain = new_url.split('/')[2]
            
            # 启动爬虫
            run_spider(new_url, allowed_domain)
            
            # 清理窗口，回到初始页面
            driver.close()
            driver.switch_to.window(driver.window_handles[1])
            driver.close()
            driver.switch_to.window(original_handle)
            
        except Exception as e:
            print(f"处理第{idx+1}个条目出错: {str(e)}")
            # 出错后强制清理所有非初始窗口
            for handle in driver.window_handles:
                if handle != original_handle:
                    driver.switch_to.window(handle)
                    driver.close()
            driver.switch_to.window(original_handle)
            continue
finally:
    driver.quit()

关键修复点 #

1. 用CrawlerRunner替代CrawlerProcess：基于Twisted reactor实现非阻塞运行，支持多次启动爬虫。
2. Spider改为实例化传参：通过__init__接收start_url和allowed_domain，避免类属性状态污染。
3. 补充CloseSpider导入：修复NameError异常。
4. Selenium改用显式等待：替换time.sleep，提升页面交互稳定性，避免元素定位失败。
5. 完善异常处理逻辑：出错时自动清理多余窗口，保证循环能继续执行。
6. 循环中重新定位元素：解决可能出现的stale element reference异常。

内容的提问来源于stack exchange，提问作者linus otte