Selenium如何实现分页导航操作，提取全部分页页面的所有数据

场景说明 #

你当前的脚本仅能提取单页表格数据，目标站点分页采用ASP.NET的__doPostBack触发异步请求的模式，需要适配该分页逻辑遍历所有页面完成全量数据采集。

完整可运行代码 #

from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from datetime import date,timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import sqlite3

# 基础配置
today = date.today()
yesterday = today - timedelta(days=2)
d3 = yesterday.strftime("%m-%d-%Y")
URL = 'https://www.dibbs.bsm.dla.mil//rfq/rfqdates.aspx?category=recent'
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(ChromeDriverManager().install(),options=options)
wait = WebDriverWait(driver, 10)

# 存储所有页面数据的总列表
all_rows = []

try:
    # 进入目标列表页
    driver.get(URL)
    driver.find_element(By.ID, "butAgree").click()
    driver.find_element(By.PARTIAL_LINK_TEXT, d3).click()
    time.sleep(2)

    while True:
        # 提取当前页所有列数据
        col1 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[1]')
        col2 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[2]')
        col3 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[3]')
        col4 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[4]')
        col5 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[5]')
        col6 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[6]')
        col7 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[7]')
        col8 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[8]')
        col9 = driver.find_elements(By.XPATH, '//table[@id="ctl00_cph1_grdRfqSearch"]/tbody/tr/td[9]')

        # 逐行合并数据存入总列表
        for i in range(len(col1)):
            row_data = [
                col1[i].text, col2[i].text, col3[i].text, col4[i].text,
                col5[i].text, col6[i].text, col7[i].text, col8[i].text, col9[i].text
            ]
            all_rows.append(row_data)
        
        # 定位下一页按钮，找不到则说明已经到最后一页
        try:
            # 当前激活页用span标签包裹，找它下一个同级td里的a标签就是下一页入口
            current_page_ele = driver.find_element(By.XPATH, '//tr[@class="pagination"]//span')
            next_page_td = current_page_ele.find_element(By.XPATH, './parent::td/following-sibling::td[1]')
            next_page_btn = next_page_td.find_element(By.TAG_NAME, 'a')
        except:
            break

        # 点击下一页，等待页面加载完成
        next_page_btn.click()
        wait.until(EC.staleness_of(current_page_ele))
        time.sleep(1)

finally:
    driver.quit()

# 后续可自行将all_rows写入SQLite或CSV文件
print(f"共采集到{len(all_rows)}条数据")

核心逻辑说明 #

• 循环控制分页遍历，每次提取完当前页数据后尝试定位下一页入口
• 利用当前激活页的span标签作为锚点定位下一页按钮，适配分页跳转后DOM的动态变化
• 组合使用显式等待和短时间强制等待，避免异步PostBack请求未完成导致的元素定位失败
• 所有页面数据统一存入all_rows列表，后续可直接对接你原本需要的存储逻辑

内容的提问来源于stack exchange，提问作者Rikky Bhai