基于Selenium的Python版LinkedIn职位机器人滚动加载问题求助

原代码仅能获取页面初始可见的7个职位，要实现动态加载全部职位，核心是模拟滚动职位列表容器触发加载，直到没有新职位出现。以下是具体修改方案：

关键实现逻辑 #

• 定位LinkedIn职位列表的专属滚动容器（而非整个页面），避免干扰其他页面元素
• 循环滚动容器到底部，等待新职位加载
• 对比滚动前后的职位数量，判断是否已加载全部结果

修改后的完整代码 #

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.action_chains import ActionChains, ScrollOrigin
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os
import time
import random

URL = "https://www.linkedin.com/jobs/search/?currentJobId=3483760254&keywords=python%20developer&refresh=true"
EMAIL = os.environ.get("MYEMAIL")
PASSWORD = os.environ.get("MYPASSWORD")

# Setup the chrome driver
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
# 可选：添加防检测配置
# options.add_argument("--headless=new")
# options.add_argument("--disable-blink-features=AutomationControlled")
service = Service("/home/hoods/Documents/development/chromedriver")
driver = webdriver.Chrome(service=service, options=options)

driver.get(URL)
driver.maximize_window()

# Sign in
sign_in = driver.find_element(By.XPATH, "/html/body/div[3]/a[1]")
time.sleep(random.uniform(2, 3))
sign_in.click()
time.sleep(random.uniform(2, 3))

username = driver.find_element(By.ID, "username")
username.send_keys(EMAIL)
password = driver.find_element(By.ID, "password")
password.send_keys(PASSWORD)
login_form_button = driver.find_element(By.CSS_SELECTOR, ".login__form button")
login_form_button.click()
time.sleep(random.uniform(3, 5))

# Minimize chat window
try:
    minimize_message = driver.find_element(By.XPATH, '//*[@id="msg-overlay"]/div[1]/header/div[2]')
    minimize_message.click()
    time.sleep(random.uniform(1, 2))
except NoSuchElementException:
    pass

# Select easy apply filter
easy_apply_filter = driver.find_element(By.XPATH, '//*[@id="search-reusables__filters-bar"]/ul/li[8]/div')
easy_apply_filter.click()
time.sleep(random.uniform(2, 3))

# -------------------------- 新增滚动加载逻辑 --------------------------
def load_all_jobs():
    # 定位职位列表容器
    job_list_container = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, ".scaffold-layout__list-container"))
    )
    last_job_count = 0
    while True:
        # 获取当前职位数量
        current_jobs = driver.find_elements(By.CSS_SELECTOR, ".job-card-container--clickable")
        current_count = len(current_jobs)
        if current_count == last_job_count:
            # 职位数量不再增加，说明已加载全部
            break
        last_job_count = current_count
        
        # 滚动容器到底部
        actions = ActionChains(driver)
        scroll_origin = ScrollOrigin.from_element(job_list_container)
        actions.scroll_from_origin(scroll_origin, 0, 10000).perform()
        
        # 随机等待，模拟人浏览的延迟
        time.sleep(random.uniform(2, 4))
        
        # 检查是否出现"没有更多结果"的提示
        try:
            no_more_results = driver.find_element(By.CSS_SELECTOR, ".jobs-search-no-results-banner")
            if no_more_results.is_displayed():
                break
        except NoSuchElementException:
            pass

# 执行滚动加载所有职位
load_all_jobs()

# 获取全部职位列表
jobs = driver.find_elements(By.CSS_SELECTOR, ".job-card-container--clickable")
print(f"共加载到 {len(jobs)} 个职位")
# -------------------------- 滚动加载逻辑结束 --------------------------

# 遍历职位执行收藏和关注
for index, job in enumerate(jobs):
    if index == 0:
        continue  # 跳过第一个职位
    try:
        # 点击职位（确保元素可点击）
        WebDriverWait(driver, 5).until(EC.element_to_be_clickable(job)).click()
        time.sleep(random.uniform(1, 2))
        
        # 点击收藏按钮
        save_button = WebDriverWait(driver, 5).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, ".jobs-save-button.artdeco-button.artdeco-button--3.artdeco-button--secondary"))
        )
        save_button.click()
        time.sleep(random.uniform(1, 2))
        
        # 点击关注按钮
        follow_button = WebDriverWait(driver, 5).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, ".follow.artdeco-button.artdeco-button--secondary.ml5"))
        )
        follow_button.click()
        time.sleep(random.uniform(1, 2))
        
    except (NoSuchElementException, TimeoutException):
        print(f"处理第 {index+1} 个职位时出错，跳过")
        continue

driver.quit()

重要说明 #

• 用random.uniform()替代固定sleep，降低被LinkedIn反爬机制检测的概率
• 使用WebDriverWait显式等待替代固定等待，提升代码稳定性
• 增加异常捕获，避免单个职位处理失败导致程序终止
• 可根据需求启用注释中的防检测配置，如无头模式、禁用自动化标识

内容的提问来源于stack exchange，提问作者Mathew Still