使用Selenium抓取内页内容后无法循环点击下一页的技术求助

核心修改点 #

• 重构链接收集逻辑，加入循环翻页机制，持续抓取所有页面的职位链接
• 优化内容抓取函数的复用性，避免重复操作浏览器实例
• 完善翻页终止判断，无下一页按钮时自动停止循环

完整代码实现 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

base_link = 'https://www.indeed.com/jobs?q=developer&sc=0kf%3Aattr%28DSQF7%29%3B&start=0'

def get_all_job_links(driver):
    all_links = []
    while True:
        # 等待当前页面职位卡片加载完成
        job_cards = WebDriverWait(driver, 20).until(
            EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".job_seen_beacon"))
        )
        # 提取当前页面所有职位链接
        for card in job_cards:
            title_link = card.find_element(By.CSS_SELECTOR, "h2 > a[class^='jcs-JobTitle']").get_attribute("href")
            all_links.append(title_link)
        
        # 尝试点击下一页
        try:
            next_page_btn = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, "a[aria-label='Next Page']"))
            )
            # 滚动到按钮位置避免遮挡
            driver.execute_script("arguments[0].scrollIntoView(true);", next_page_btn)
            next_page_btn.click()
            # 等待页面刷新完成
            WebDriverWait(driver, 20).until(
                EC.staleness_of(job_cards[0])
            )
        except Exception:
            # 无下一页时退出循环
            break
    return all_links

def get_job_content(driver, link):
    driver.get(link)
    # 抓取职位名称
    title = WebDriverWait(driver, 20).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "h1.jobsearch-JobInfoHeader-title"))
    ).text
    # 抓取所需技能
    try:
        skill_section = driver.find_element(
            By.XPATH, "//*[@id='jobDescriptionText']//div[./div/b[contains(.,'Required Skills')]]"
        )
        skill = skill_section.get_attribute("textContent").strip()
    except Exception:
        skill = "未找到明确的Required Skills部分"
    return {"职位名称": title, "所需技能": skill}

if __name__ == '__main__':
    with webdriver.Chrome() as driver:
        driver.get(base_link)
        job_links = get_all_job_links(driver)
        # 遍历所有职位链接抓取内容
        for idx, link in enumerate(job_links, 1):
            print(f"正在处理第{idx}个职位...")
            job_info = get_job_content(driver, link)
            print(job_info)
            print("-" * 50)

关键说明 #

• 翻页逻辑：通过while True循环持续抓取，每次完成当前页面链接收集后尝试点击下一页，捕获异常时终止循环
• 页面加载验证：使用staleness_of判断页面是否刷新，确保新页面的职位卡片已加载完成
• 点击稳定性优化：添加滚动到按钮的操作，避免因页面元素遮挡导致点击失败
• 异常兼容：对没有明确Required Skills的职位返回提示文本，防止程序中断

内容的提问来源于stack exchange，提问作者robots.txt