使用Selenium点击下一页后无法获取新HTML代码，URL无变化

爬取https://www.coworker.com/search/turkey/izmir时，因页面是JavaScript渲染选择Selenium，但点击下一页后URL无变化，driver获取的page_source始终未更新，测试代码如下：

import requests
import xlsxwriter
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep
spaces = []

kingUrl = f"https://www.coworker.com/search/turkey/izmir"
driver = webdriver.Chrome()
#wait = WebDriverWait(driver, 10)
driver.get(kingUrl)
page = 0
count = 0
while page != 2:
    sleep(5)
    html = driver.page_source
    # print(html)

    soup = BeautifulSoup(html, "html.parser")
    current_page_number = driver.find_element(By.CSS_SELECTOR,
                                              '#search_results > div > div.col-12.space-pagination-outer.search-pagination-outer > nav > ul > li.page-item.active > span').text
    print(current_page_number)
    tags = soup.find_all("a", class_="optimizely-review-trigger")
    # print(tags)
    for item in tags:
        count += 1
        spaces.append(item['href'])

    page += 1
    if page != 1:
        driver.execute_script(
            "window.scrollTo(0, document.body.scrollHeight - 2300);")
        sleep(1)
    # click_button = driver.find_element(
    # by=By.CLASS_NAME, value="page-link search-page-link")
    # click_button.click()
        button = driver.find_element("xpath",
                                     '//*[@id="search_results"]/div/div[11]/nav/ul/li[4]/a')
        button.click()

        WebDriverWait(driver, 100).until(lambda driver: driver.find_element(By.CSS_SELECTOR,
                                                                            '#search_results > div > div.col-12.space-pagination-outer.search-pagination-outer > nav > ul > li.page-item.active > span').text != current_page_number)
        sleep(100)
        # wait.until(EC.presence_of_element_located(
        # (By.CLASS_NAME, "sr-only")))
        # wait.until(EC.staleness_of())
        #driver.implicitly_wait(100)
        print(current_page_number)
        # sleep(10)

问题分析与修复要点 #

1. 分页按钮定位硬编码 #

你用固定XPath定位下一页按钮，依赖页面固定层级，一旦页面结构变动就会失效。改用通用定位方式，比如通过按钮的aria-label或文本定位：

# 通过aria-label包含"Next"定位
button = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, 'a.page-link.search-page-link[aria-label*="Next"]'))
)
# 或通过文本定位
button = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.XPATH, '//a[contains(text(), "Next")]'))
)

2. 等待条件逻辑错误 #

当前仅判断页码文本变化，无法确保页面内容已更新。应等待当前页面的列表元素失效，确认新内容已替换旧内容：

# 点击前获取当前页面第一个列表元素
first_space = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, 'a.optimizely-review-trigger'))
)
button.click()
# 等待旧元素失效，说明页面已更新
WebDriverWait(driver, 10).until(EC.staleness_of(first_space))

3. 滚动操作优化 #

固定滚动位置没必要，改为滚动到按钮可视区域即可：

driver.execute_script("arguments[0].scrollIntoView(true);", button)
sleep(0.5)  # 给滚动动画留时间

4. 循环逻辑优化 #

固定页数的循环不灵活，改为判断下一页按钮是否可点击，直到没有下一页为止：

while True:
    # 爬取当前页面逻辑...
    
    # 尝试获取下一页按钮，失败则退出循环
    try:
        next_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, 'a.page-link.search-page-link[aria-label*="Next"]'))
        )
    except:
        break
    # 点击下一页并等待更新...

修改后的完整代码 #

import xlsxwriter
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep

spaces = []
kingUrl = "https://www.coworker.com/search/turkey/izmir"
driver = webdriver.Chrome()
driver.get(kingUrl)
count = 0

while True:
    # 等待页面列表元素加载完成
    WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'a.optimizely-review-trigger'))
    )
    html = driver.page_source
    soup = BeautifulSoup(html, "html.parser")
    
    # 获取并打印当前页码
    current_page_number = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, '#search_results li.page-item.active > span'))
    ).text
    print(f"当前爬取页码: {current_page_number}")
    
    # 提取所有空间链接
    tags = soup.find_all("a", class_="optimizely-review-trigger")
    for item in tags:
        count += 1
        spaces.append(item['href'])
    
    # 处理下一页
    try:
        next_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, 'a.page-link.search-page-link[aria-label*="Next"]'))
        )
        # 滚动到按钮位置
        driver.execute_script("arguments[0].scrollIntoView(true);", next_button)
        sleep(0.5)
        # 记录当前第一个元素用于等待更新
        first_space = driver.find_element(By.CSS_SELECTOR, 'a.optimizely-review-trigger')
        next_button.click()
        # 等待旧元素失效，确认页面更新
        WebDriverWait(driver, 10).until(EC.staleness_of(first_space))
    except:
        print("已爬取所有页面")
        break

print(f"共爬取{count}条数据")
driver.quit()

内容的提问来源于stack exchange，提问作者Mohi Eldin