Selenium Python爬虫仅抓取分页网站第一页链接问题排查

我用Python的Selenium开发网页爬虫，目标是抓取分页网站的多页链接。已经配置了分页导航和链接提取，但代码仅抓取第一页链接后就关闭浏览器，无法继续后续页面的抓取。已确认代码能识别并点击“下一页”按钮，但点击后无法继续抓取，浏览器随即关闭。

原代码如下：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup

def scrape_links(url):
    firefox_options = Options()
    firefox_options.binary_location = "/usr/lib/firefox-esr/firefox-bin"
    driver = webdriver.Firefox(options=firefox_options)

    all_hrefs = []
    current_page = 1

    while True:
        driver.get(url)
        html_content = driver.page_source
        soup = BeautifulSoup(html_content, "html.parser")
        play_links = soup.select(
            "section.main article.page-detail div.content-wrapper div.episode-list-wrapper div.episode-list div.episode-wrap div.episode div.episode-snapshot a.play"
        )
        hrefs = [link.get("href") for link in play_links]
        all_hrefs.extend(hrefs)
        # Find the "next page" button
        try:
            next_page_button = driver.find_element(By.CSS_SELECTOR, "section.main article.page-detail div.content-wrapper div.episode-list-wrapper nav ul.pagination li.page-item:not(.disabled) a.page-link")

            if next_page_button:
                next_page_button.click()
                current_page += 1
                url = f"{url}?page={current_page}"
            else:
                break 
        except:
            break

    title_span = soup.select_one(
        "section.main article.page-detail div.header-wrapper header.anime-header div.title-wrapper h1 span"
    )
    website_title = title_span.get_text() if title_span else "Unknown Title"

    driver.quit()

    with open(f"{website_title}.txt", "w") as file:
        for href in all_hrefs:
            file.write("https://animepahe.ru" + href + "\n")

    print(f"Links have been saved for {website_title}")
num_links = int(input("Enter the number of animepahe webpages: "))
urls = []
for i in range(num_links):
    url = input(f"Enter URL {i+1}: ")
    urls.append(url)
for url in urls:
    scrape_links(url)

问题排查分析 #

• 重复调用driver.get(url)覆盖页面跳转：点击下一页按钮后，页面已经跳转到下一页，但循环里又执行driver.get(url)，强制回到拼接后的URL（甚至可能URL拼接错误，比如原始URL已有参数时重复添加），导致流程混乱。
• 下一页按钮定位不精准：原CSS选择器li.page-item:not(.disabled) a.page-link会匹配所有未禁用的分页按钮（比如上一页、下一页），无法精准定位“下一页”按钮，可能点击错误元素触发异常，直接跳出循环。
• 无页面加载等待机制：点击下一页后没有等待页面加载完成就开始抓取，可能导致元素未加载完全，触发异常后直接退出循环。
• 异常捕获过于宽泛：except:会捕获所有异常，包括正常的页面加载延迟，无法定位具体错误原因，导致程序直接终止循环。
• 标题获取时机错误：最后获取标题时用的是循环结束后的soup，如果循环提前终止，可能拿到的是错误页面的标题，甚至触发空指针。

修复方案及代码 #

修复后的代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time

def scrape_links(base_url):
    firefox_options = Options()
    firefox_options.binary_location = "/usr/lib/firefox-esr/firefox-bin"
    driver = webdriver.Firefox(options=firefox_options)
    driver.get(base_url)
    all_hrefs = []
    
    # 初始加载后立即获取标题，避免循环异常导致标题丢失
    try:
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "section.main article.page-detail div.header-wrapper header.anime-header div.title-wrapper h1 span"))
        )
        soup = BeautifulSoup(driver.page_source, "html.parser")
        title_span = soup.select_one("section.main article.page-detail div.header-wrapper header.anime-header div.title-wrapper h1 span")
        website_title = title_span.get_text().strip() if title_span else "Unknown Title"
    except Exception as e:
        print(f"获取标题失败: {e}")
        website_title = "Unknown Title"

    while True:
        # 等待页面核心元素加载完成
        try:
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, "section.main article.page-detail div.content-wrapper div.episode-list-wrapper div.episode-list"))
            )
            html_content = driver.page_source
            soup = BeautifulSoup(html_content, "html.parser")
            play_links = soup.select(
                "section.main article.page-detail div.content-wrapper div.episode-list-wrapper div.episode-list div.episode-wrap div.episode div.episode-snapshot a.play"
            )
            hrefs = [link.get("href") for link in play_links]
            all_hrefs.extend(hrefs)
            print(f"已抓取当前页 {len(hrefs)} 个链接")

            # 精准定位下一页按钮（根据实际页面调整XPATH，示例匹配含"Next"文本的按钮）
            try:
                next_page_button = WebDriverWait(driver, 10).until(
                    EC.element_to_be_clickable((By.XPATH, "//nav[@class='pagination']//li[contains(@class, 'page-item') and not(contains(@class, 'disabled'))]//a[contains(text(), 'Next')]"))
                )
                next_page_button.click()
                # 等待页面跳转完成，避免重复抓取
                time.sleep(1)
            except Exception as e:
                print(f"无更多页面或下一页按钮不可点击: {e}")
                break
        except Exception as e:
            print(f"页面加载或抓取失败: {e}")
            break

    driver.quit()

    # 保存链接（指定编码避免乱码）
    with open(f"{website_title}.txt", "w", encoding="utf-8") as file:
        for href in all_hrefs:
            file.write(f"https://animepahe.ru{href}\n")

    print(f"Links have been saved for {website_title}，共抓取 {len(all_hrefs)} 个链接")

num_links = int(input("Enter the number of animepahe webpages: "))
urls = []
for i in range(num_links):
    url = input(f"Enter URL {i+1}: ")
    urls.append(url)
for url in urls:
    scrape_links(url)

关键修复点说明 #

• 移除重复的driver.get(url)：初始加载页面后，通过点击下一页按钮自然跳转，不再强制刷新URL，避免覆盖页面状态。
• 精准定位下一页按钮：改用XPATH定位包含"Next"文本的下一页按钮（可根据实际页面的按钮文本/属性调整），确保只点击正确的下一页元素。
• 添加显式等待：使用WebDriverWait等待关键元素加载完成，避免因页面未加载完全导致的抓取失败。
• 调整标题获取时机：在初始加载页面后立即获取标题，避免循环终止后无法获取正确标题。
• 缩小异常捕获范围：针对不同操作单独捕获异常，方便定位问题，同时避免因小错误直接终止整个循环。
• 添加页面跳转等待：点击下一页后添加短暂等待，确保页面完成跳转后再进行下一次抓取。

内容的提问来源于stack exchange，提问作者Olasubomi