Selenium Python LinkedIn求职自动化：循环仅7次迭代即终止

我尝试用Python的Selenium WebDriver实现LinkedIn职位申请自动化，代码如下：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time


#Defines the bridge between Selenium and Chrome
s = Service ("C:/Users/ugila/Desktop/1/Application/chromedriver.exe")

#To ensure Chrome Browser does not close automatically right after opening the targetted URL
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)

#Executes bridge between selenium and chrome while ensuring setting chosen in above option of 
# browser not closing is also executed
driver = webdriver.Chrome(service = s, options=options)

#Open below website in the browser that just got opened
driver.get("https://www.linkedin.com/jobs/search/?currentJobId=3639948433&f_F=it%2Cprjm&f_JT=C&f_TPR=r2592000&geoId=105149290&keywords=IT%20Project%20Manager&location=Ontario%2C%20Canada&refresh=true&sortBy=DD")

#locate Sign in button from opened website
Sign_in_Button = driver.find_element(By.LINK_TEXT,'Sign in')

#click sign in button and have sign in page open in the same window
Sign_in_Button.click()

#Username field on sign in page
Username_field = driver.find_element(By.ID, 'username')

#Enter text in username field on sign in page
Username_field.send_keys("*My Email Address*")

#Password field on sign in page
Password_field = driver.find_element(By.ID, 'password')

#Enter text in password field on sign in page
Password_field.send_keys("*My Password*")

#Login in to Linkedin by entering the username and password provided above
Password_field.send_keys(Keys.ENTER)
time.sleep(2)

#List of all the posting in filtered job list
All_Listings = driver.find_elements(By.CSS_SELECTOR, '.job-card-container.relative.job-card-list.job-card-container--clickable')
                                   
                                   
#Loop through all the listings in filtered list
for Job in All_Listings:
    print("loop called properly")
    Job.click()
    time.sleep(2)

运行输出：

[Running] python -u "c:\Users\ugila\Desktop\1\Application\Bot.py"
loop called properly
loop called properly
loop called properly
loop called properly
loop called properly
loop called properly
loop called properly

[Done] exited with code=0 in 40.964 seconds

循环仅执行7次就停止，期望遍历页面上所有职位列表，求排查原因及修复方案。

问题原因 #

1. 动态加载限制：LinkedIn职位列表采用滚动加载机制，初始获取的All_Listings仅包含页面当前可见的7个职位卡片，未滚动到的区域元素尚未渲染，因此只能遍历到这7个元素。
2. DOM变更导致元素失效：点击职位后，页面右侧加载详情，DOM结构发生变化，之前获取的列表元素会变成StaleElementReferenceException（部分情况下可能无报错但元素不可交互），导致循环提前终止。

修复方案 #

方案1：先滚动加载所有职位再遍历 #

通过滚动页面触发所有职位加载，再获取完整列表进行遍历，同时每次循环重新获取元素避免失效：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

s = Service("C:/Users/ugila/Desktop/1/Application/chromedriver.exe")
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
driver = webdriver.Chrome(service=s, options=options)

driver.get("https://www.linkedin.com/jobs/search/?currentJobId=3639948433&f_F=it%2Cprjm&f_JT=C&f_TPR=r2592000&geoId=105149290&keywords=IT%20Project%20Manager&location=Ontario%2C%20Canada&refresh=true&sortBy=DD")

# 登录流程
Sign_in_Button = driver.find_element(By.LINK_TEXT,'Sign in')
Sign_in_Button.click()

Username_field = driver.find_element(By.ID, 'username')
Username_field.send_keys("*My Email Address*")

Password_field = driver.find_element(By.ID, 'password')
Password_field.send_keys("*My Password*")
Password_field.send_keys(Keys.ENTER)

# 显式等待职位列表加载完成
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, '.job-card-container')))

# 滚动加载所有职位
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)  # 等待新内容加载
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break  # 无更多内容加载
    last_height = new_height

# 遍历所有职位，每次重新获取元素避免失效
total_jobs = len(driver.find_elements(By.CSS_SELECTOR, '.job-card-container.relative.job-card-list.job-card-container--clickable'))
for index in range(total_jobs):
    current_job = driver.find_elements(By.CSS_SELECTOR, '.job-card-container.relative.job-card-list.job-card-container--clickable')[index]
    print(f"处理第{index+1}个职位")
    try:
        current_job.click()
        time.sleep(2)
        # 此处可添加职位申请逻辑（如点击Easy Apply等）
    except Exception as e:
        print(f"处理职位出错: {str(e)}")
        continue

方案2：边滚动边遍历（适合大数量职位） #

无需一次性加载所有职位，每次循环前重新获取当前可见列表，逐步滚动加载：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

s = Service("C:/Users/ugila/Desktop/1/Application/chromedriver.exe")
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
driver = webdriver.Chrome(service=s, options=options)

driver.get("https://www.linkedin.com/jobs/search/?currentJobId=3639948433&f_F=it%2Cprjm&f_JT=C&f_TPR=r2592000&geoId=105149290&keywords=IT%20Project%20Manager&location=Ontario%2C%20Canada&refresh=true&sortBy=DD")

# 登录流程（同方案1）
Sign_in_Button = driver.find_element(By.LINK_TEXT,'Sign in')
Sign_in_Button.click()

Username_field = driver.find_element(By.ID, 'username')
Username_field.send_keys("*My Email Address*")

Password_field = driver.find_element(By.ID, 'password')
Password_field.send_keys("*My Password*")
Password_field.send_keys(Keys.ENTER)

wait = WebDriverWait(driver, 10)
processed_count = 0

while True:
    # 获取当前可见的职位列表
    listings = driver.find_elements(By.CSS_SELECTOR, '.job-card-container.relative.job-card-list.job-card-container--clickable')
    # 处理未遍历过的职位
    for i in range(processed_count, len(listings)):
        job = listings[i]
        print(f"处理第{i+1}个职位")
        try:
            job.click()
            time.sleep(2)
            processed_count += 1
            # 此处可添加申请逻辑
        except Exception as e:
            print(f"出错: {str(e)}")
            continue
    # 滚动加载更多职位
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)
    # 检查是否有新职位加载
    new_listings_count = len(driver.find_elements(By.CSS_SELECTOR, '.job-card-container.relative.job-card-list.job-card-container--clickable'))
    if new_listings_count == processed_count:
        break  # 无更多职位

关键优化点 #

• 用WebDriverWait显式等待替代time.sleep，提升代码稳定性（等待元素加载完成再操作）。
• 处理LinkedIn的滚动加载机制，确保获取所有职位。
• 每次循环重新获取元素，避免因DOM变更导致的StaleElementReferenceException。

内容的提问来源于stack exchange，提问作者Tanzeel Gilani