URL无变化时如何编写多页奥运运动员数据爬取代码？

该站点属于单页应用，分页数据通过前端动态加载，URL不会随页码变化，解决思路是定位分页控件，模拟人工点击翻页操作，结合原有爬取逻辑实现多页数据采集。

修改后完整代码 #

import chromedriver_autoinstaller
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time

# 配置爬取的起始页和结束页
START_PAGE = 1
END_PAGE = 5  # 示例：爬取第1到第5页，可自行修改

# Chrome Driver setup
chromedriver_autoinstaller.install()
driver = webdriver.Chrome()
driver.get("https://olympics.com/en/paris-2024/athletes")
driver.implicitly_wait(10)

# 处理Cookie弹窗
try:
    cookies_btn = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.CSS_SELECTOR, "#onetrust-accept-btn-handler"))
    )
    cookies_btn.click()
    print("已接受Cookie。")
except Exception as e:
    print("接受Cookie失败:", str(e))

time.sleep(3)

# 存储所有运动员数据
athletes_data = []

def crawl_current_page():
    """爬取当前页面的50位运动员数据"""
    for i in range(1, 51):
        try:
            # 定位运动员卡片
            athlete_card_xpath = f"//table[@id='mirs-table-athletes']//tr[{i}]//td[1]//a"
            athlete_card = WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, athlete_card_xpath))
            )
            driver.execute_script("arguments[0].scrollIntoView();", athlete_card)
            time.sleep(1)

            # 点击进入详情页
            driver.execute_script("arguments[0].click();", athlete_card)

            # 采集核心信息
            name = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.XPATH, "//*[@id='PersonInfo']/div/div/div[2]/div[1]/div[1]/b"))
            ).text
            nationality = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.XPATH, "//*[@id='PersonInfo']/div/div/div[2]/div[1]/div[2]/div"))
            ).text
            discipline = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.XPATH, "//*[@id='PersonInfo']/div/div/div[2]/div[1]/div[3]/div/span"))
            ).text
            age = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.XPATH, "//*[@id='PersonInfo']/div/div/div[2]/div[2]/div[1]/div[2]"))
            ).text.replace("Age:", "").strip()
            gender = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.XPATH, "//*[@id='PersonInfo']/div/div/div[2]/div[2]/div[1]/div[3]"))
            ).text.replace("Gender:", "").strip()

            # 采集附加信息（处理元素不存在的情况）
            rank = None
            try:
                rank = WebDriverWait(driver, 3).until(
                    EC.presence_of_element_located((By.XPATH, "//*[@id='mirs-table-biomedals']/tbody/tr/td[3]"))
                ).text
            except (TimeoutException, NoSuchElementException):
                pass

            medal = None
            try:
                medal_element = WebDriverWait(driver, 3).until(
                    EC.presence_of_element_located((By.XPATH, "//*[@id='mirs-table-biomedals']/tbody/tr/td[4]/svg/use"))
                )
                medal = medal_element.get_attribute("xlink:href")
            except (TimeoutException, NoSuchElementException):
                pass

            debut = None
            try:
                debut = WebDriverWait(driver, 3).until(
                    EC.presence_of_element_located((By.XPATH, "//*[@id='BiographicalInformation']/div[2]/div[10]/p[2]"))
                ).text
            except (TimeoutException, NoSuchElementException):
                pass

            occupation = None
            try:
                occupation = WebDriverWait(driver, 3).until(
                    EC.presence_of_element_located((By.XPATH, "//*[@id='BiographicalInformation']/div[2]/div[16]/p[2]"))
                ).text
            except (TimeoutException, NoSuchElementException):
                pass

            # 追加数据到列表
            athletes_data.append([name, nationality, discipline, age, gender, rank, medal, debut, occupation])

            # 返回列表页
            time.sleep(5)
            driver.back()
            time.sleep(5)

        except Exception as e:
            print(f"采集第{current_page}页第{i}位运动员失败: {str(e)}")
            try:
                driver.back()
            except:
                pass
            time.sleep(3)

# 分页爬取逻辑
for current_page in range(START_PAGE, END_PAGE + 1):
    print(f"开始爬取第{current_page}页...")
    crawl_current_page()
    
    # 如果不是最后一页，点击下一页
    if current_page != END_PAGE:
        try:
            # 定位下一页按钮（可根据实际页面结构调整选择器）
            next_page_btn = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, "button.pagination-next"))
            )
            # 滚动到分页控件位置
            driver.execute_script("arguments[0].scrollIntoView();", next_page_btn)
            time.sleep(2)
            next_page_btn.click()
            # 等待页面加载完成（等待表格更新）
            WebDriverWait(driver, 15).until(
                EC.staleness_of(driver.find_element(By.ID, "mirs-table-athletes"))
            )
            time.sleep(3)
            print(f"已切换到第{current_page + 1}页")
        except Exception as e:
            print(f"切换到下一页失败: {str(e)}")
            break

# 关闭浏览器
driver.quit()

# 保存数据到Excel
df = pd.DataFrame(athletes_data, columns=["Name", "Nationality", "Discipline", "Age", "Gender", "Rank", "Medal", "Debut", "Occupation"])
df.to_excel("multi_page_athletes_data.xlsx", index=False)
print(f"共采集{len(athletes_data)}条运动员数据，已保存到multi_page_athletes_data.xlsx")

关键说明 #

1. 分页控件定位：代码中使用button.pagination-next作为下一页按钮选择器，若页面结构变化，需打开浏览器开发者工具查看实际的分页按钮CSS选择器或XPATH，替换即可。
2. 页面加载等待：使用EC.staleness_of等待原表格元素失效，确保新页面数据已加载完成，避免爬取重复数据。
3. 异常处理优化：对附加信息的采集单独做异常捕获，避免单个字段缺失导致整行数据丢失；同时处理返回列表页的异常情况。
4. 可配置参数：通过START_PAGE和END_PAGE设置爬取的页码范围，灵活控制采集区间。

内容的提问来源于stack exchange，提问作者하비비