Selenium处理网页随机弹出订阅弹窗问题求助

问题描述 #

爬取Centris.ca房源页面时，脚本执行期间会随机弹出订阅弹窗，弹窗出现时机完全不确定：可能在翻页前、爬取房源价格时触发，甚至全程不出现。手动访问时弹窗频率极低，但脚本运行时频繁出现，现有仅在翻页后检查弹窗的逻辑无法覆盖所有场景，导致爬虫经常因弹窗遮挡元素而失败。

解决方案 #

针对随机弹窗的问题，核心思路是在每次元素交互前主动检查并关闭弹窗，同时优化爬虫行为模拟真实用户，降低弹窗触发概率：

• 封装通用弹窗检查函数，在任何可能触发弹窗的操作前调用
• 替换固定time.sleep()为Selenium显式等待，提高效率同时确保元素加载完成
• 添加Chrome配置模拟真实浏览器环境，降低被识别为爬虫的概率
• 增加异常捕获兜底，即使弹窗漏检也能尝试关闭后重试操作

修改后的完整代码 #

import numpy as np
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException

url = 'https://www.centris.ca/en/properties~for-sale~brossard?view=Thumbnail'
centris_list = []

def close_subscription_popup(driver):
    """检查并关闭订阅弹窗"""
    try:
        close_btn = WebDriverWait(driver, 2).until(
            EC.element_to_be_clickable((By.XPATH, ".//div[@class='DialogInsightLightBoxCloseButton']"))
        )
        close_btn.click()
        print('已关闭订阅弹窗')
    except:
        # 无弹窗时直接跳过
        pass

def scrap_pages(driver):
    close_subscription_popup(driver)
    # 等待房源列表加载完成
    listings = WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, 'description'))
    )

    # 过滤空元素
    if listings and listings[-1].text.split('\n')[0] == '':
        del listings[-1]

    for listing in listings:
        close_subscription_popup(driver)
        try:
            price = listing.find_element(By.XPATH, ".//div[@class='price']/meta[@itemprop='price']").get_attribute('content')
            mls = listing.find_element(By.XPATH, ".//div[@id='MlsNumberNoStealth']/p").text
            prop_type = listing.find_element(By.XPATH, ".//div[@class='location-container']/span[@itemprop='category']").text
            addr = listing.find_element(By.XPATH, ".//div[@class='location-container']/span[@class='address']").text
            addr_lines = addr.split('\n')
            city = addr_lines[1] if len(addr_lines)>=2 else 'NA'
            sector = addr_lines[2] if len(addr_lines)>=3 else 'NA'
            
            bedrooms = 'NA'
            bathrooms = 'NA'
            if prop_type not in ['Land for sale', 'Lot for sale']:
                bedrooms = listing.find_element(By.XPATH, ".//div[@class='cac']").text
                bathrooms = listing.find_element(By.XPATH, ".//div[@class='sdb']").text

            listing_item = {
                'mls': mls,
                'price': price,
                'Address': addr,
                'property Type': prop_type,
                'city': city,
                'bedrooms': bedrooms,
                'bathrooms': bathrooms,
                'sector': sector
            }

            centris_list.append(listing_item)
        except ElementClickInterceptedException:
            # 元素被弹窗遮挡，关闭后重试
            close_subscription_popup(driver)
            print('元素被弹窗遮挡，已重试')
        except NoSuchElementException as e:
            print(f"定位元素失败: {e}")
            continue

if __name__ == '__main__':
    chrome_options = Options()
    chrome_options.add_experimental_option("detach", True)
    # 模拟真实浏览器UA
    chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
    # 禁用自动化检测标识
    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option('useAutomationExtension', False)
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")

    driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
    driver.get(url) 

    # 获取总页数
    total_pages = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'pager-current'))
    ).text.split('/')[1].strip() 
    
    for i in range(1, int(total_pages)):
        scrap_pages(driver)
        # 点击下一页，处理弹窗遮挡情况
        try:
            next_btn = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, 'li.next> a'))
            )
            next_btn.click()
        except ElementClickInterceptedException:
            close_subscription_popup(driver)
            next_btn = WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, 'li.next> a'))
            )
            next_btn.click()
        # 等待页面切换完成
        WebDriverWait(driver, 10).until(
            EC.staleness_of(listings[0]) if 'listings' in locals() else EC.presence_of_element_located((By.CLASS_NAME, 'description'))
        )

    print(f"爬取完成，共获取{len(centris_list)}条房源信息")
    driver.quit()

关键修改说明 #

• 弹窗检查逻辑：close_subscription_popup函数用2秒显式等待检查弹窗，不影响正常流程，确保弹窗出现即被关闭
• 元素加载优化：所有列表和按钮定位都用WebDriverWait等待元素就绪，避免页面未加载完成导致的错误
• 异常兜底：捕获元素遮挡和未找到异常，单个房源爬取失败不中断整个脚本，提升稳定性
• 反爬配置：添加真实用户代理、禁用自动化标识，降低网站对爬虫的识别度，减少弹窗触发频率

内容的提问来源于stack exchange，提问作者D.Zou