使用Requests、Beautiful Soup和Selenium爬取Glassdoor出现大量重复条目排查

问题背景 #

用Requests、Beautiful Soup和Selenium爬取Glassdoor美国地区的Data Analyst职位信息，代码能正常运行，但爬取30页后发现870条结果中有690条是重复内容。已确认Selenium确实跳转到了新页面，且未使用无头模式，想搞清楚是Glassdoor反爬机制导致，还是代码存在问题。

原代码 #

def glassdoor_scraper(url):
    
    driver = webdriver.Chrome()
    driver.get(url)
    time.sleep(10)
    
    # 定位搜索框并输入关键词
    jobs_search_title = driver.find_element(By.ID, 'KeywordSearch')
    jobs_search_title.send_keys('Data Analyst')
    jobs_search_location = driver.find_element(By.ID, 'LocationSearch')
    
    time.sleep(1)
    
    jobs_search_location.clear()
    jobs_search_location.send_keys('United States')
    click_search = driver.find_element(By.ID, 'HeroSearchButton')
    click_search.click()
    
    for page_num in range(1,10):
        time.sleep(10)
        
        res = requests.get(driver.current_url)
        soup = BeautifulSoup(res.text,'html.parser')
        
        time.sleep(2)

        companies = soup.select('.css-l2wjgv.e1n63ojh0.jobLink')
        for company in companies:
            companies_list.append(company.text)
    
        positions = soup.select('.jobLink.css-1rd3saf.eigr9kq2')
        for position in positions:
            positions_list.append(position.text)
    
        locations = soup.select('.css-l2fjlt.pr-xxsm.css-iii9i8.e1rrn5ka0')
        for location in locations:
            locations_list.append(location.text)
    
        job_post = soup.select('.eigr9kq3')
        for job in job_post:
            salary_info = job.select('.e1wijj242')
            if len(salary_info) > 0:
                for salary in salary_info:
                    salaries_list.append(salary.text)
            else:
                salaries_list.append('Salary Not Found')
    
        ratings = soup.select('.e1rrn5ka3')
        for index, rating in enumerate(ratings):
            if len(rating.text) > 0:
                ratings_list.append(rating.text)
            else:
                ratings_list.append('Rating Not Found')
        
        
        next_page = driver.find_elements(By.CLASS_NAME, 'e13qs2073')[1]
        next_page.click()
        time.sleep(5)
        try:
            close_jobalert_popup = driver.find_element(By.CLASS_NAME, 'modal_closeIcon')
        except:
            pass
        else:
            time.sleep(1)
            close_jobalert_popup.click()        
        continue
    
    #driver.close()
    print(f'{len(companies_list)} jobs found for you!')
    
    global glassdoor_dataset
    
    glassdoor_dataset = pd.DataFrame(
    {'Company Name': companies_list,
     'Company Rating': ratings_list,
     'Position Title': positions_list,
     'Location' : locations_list,
     'Est. Salary' : salaries_list
    })
    
    glassdoor_dataset.to_csv(r'glassdoor_jobs_scraped.csv')

问题根源分析 #

1. 核心代码逻辑错误（主要原因） #

你用requests.get(driver.current_url)发起独立HTTP请求，完全没带上Selenium浏览器里的会话Cookie。Glassdoor页面依赖会话验证且动态渲染，requests拿到的内容和Selenium当前显示的页面完全不一致——大概率每次返回的都是第一页内容，直接导致重复条目暴增。

2. 其他潜在问题 #

• 全局列表残留旧数据：companies_list等全局变量如果多次调用函数，会累积之前的爬取结果，加剧重复。
• 动态class选择器不稳定：Glassdoor的class是动态生成的，可能某次爬取时匹配到重复内容。
• 翻页逻辑风险高：driver.find_elements(By.CLASS_NAME, 'e13qs2073')[1]直接取索引1的元素，页面结构一变就会出错，甚至点击错误元素导致页面不跳转。

修复方案 #

1. 替换requests请求，用Selenium页面源码 #

删掉requests请求逻辑，直接用driver.page_source获取当前浏览器的页面内容，保证会话一致，拿到的是当前页面真实数据：

# 替换原有的res和soup初始化代码
soup = BeautifulSoup(driver.page_source, 'html.parser')

2. 函数内部初始化列表，避免全局污染 #

在函数开头定义所有存储列表，不用全局变量：

def glassdoor_scraper(url):
    companies_list = []
    positions_list = []
    locations_list = []
    salaries_list = []
    ratings_list = []
    # 后续代码不变...

3. 使用稳定的data-test属性选择器 #

Glassdoor大部分元素都有data-test属性，比动态class可靠得多，比如：

companies = soup.select('[data-test="employer-name"]')
positions = soup.select('[data-test="job-title"]')
locations = soup.select('[data-test="emp-location"]')

4. 优化翻页和弹窗处理 #

• 先判断下一页元素是否存在，再点击，避免索引越界：

next_pages = driver.find_elements(By.CLASS_NAME, 'e13qs2073')
if len(next_pages) >= 2:
    next_pages[1].click()
else:
    print("已到最后一页，停止爬取")
    break

• 用显式等待处理弹窗，比固定sleep更可靠：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

try:
    close_btn = WebDriverWait(driver, 5).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'modal_closeIcon'))
    )
    close_btn.click()
except:
    pass

5. 反爬优化 #

• 用随机延迟代替固定sleep，避免被识别为爬虫：

import random
time.sleep(random.randint(5, 10))

• 给ChromeDriver设置自定义User-Agent：

from selenium.webdriver.chrome.options import Options

options = Options()
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
driver = webdriver.Chrome(options=options)

修改后的完整代码示例 #

import time
import random
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup

def glassdoor_scraper(url):
    # 初始化存储列表
    companies_list = []
    positions_list = []
    locations_list = []
    salaries_list = []
    ratings_list = []
    
    # 配置ChromeDriver
    options = Options()
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
    driver = webdriver.Chrome(options=options)
    driver.get(url)
    time.sleep(random.randint(8, 12))
    
    # 搜索职位和地点
    jobs_search_title = driver.find_element(By.ID, 'KeywordSearch')
    jobs_search_title.send_keys('Data Analyst')
    jobs_search_location = driver.find_element(By.ID, 'LocationSearch')
    
    time.sleep(random.randint(1, 3))
    
    jobs_search_location.clear()
    jobs_search_location.send_keys('United States')
    click_search = driver.find_element(By.ID, 'HeroSearchButton')
    click_search.click()
    time.sleep(random.randint(8, 12))
    
    for page_num in range(1, 31):  # 爬30页
        # 获取当前页面源码
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        time.sleep(random.randint(2, 4))

        # 提取公司名称
        companies = soup.select('[data-test="employer-name"]')
        for company in companies:
            companies_list.append(company.text.strip())
    
        # 提取职位标题
        positions = soup.select('[data-test="job-title"]')
        for position in positions:
            positions_list.append(position.text.strip())
    
        # 提取地点
        locations = soup.select('[data-test="emp-location"]')
        for location in locations:
            locations_list.append(location.text.strip())
    
        # 提取薪资信息
        job_posts = soup.select('[data-test="jobListing"]')
        for job in job_posts:
            salary_info = job.select('[data-test="detailSalary"]')
            if salary_info:
                salaries_list.append(salary_info[0].text.strip())
            else:
                salaries_list.append('Salary Not Found')
    
        # 提取公司评分
        ratings = soup.select('[data-test="rating"]')
        for rating in ratings:
            if rating.text.strip():
                ratings_list.append(rating.text.strip())
            else:
                ratings_list.append('Rating Not Found')
        
        # 处理翻页
        try:
            next_pages = driver.find_elements(By.CLASS_NAME, 'e13qs2073')
            if len(next_pages) >= 2:
                next_pages[1].click()
                time.sleep(random.randint(5, 8))
            else:
                print("已到最后一页，停止爬取")
                break
        except Exception as e:
            print(f"翻页出错: {e}")
            break
        
        # 处理弹窗
        try:
            close_btn = WebDriverWait(driver, 5).until(
                EC.presence_of_element_located((By.CLASS_NAME, 'modal_closeIcon'))
            )
            close_btn.click()
            time.sleep(random.randint(1, 2))
        except:
            pass
        
        print(f"已完成第 {page_num} 页爬取")
    
    driver.close()
    print(f'{len(companies_list)} jobs found for you!')
    
    glassdoor_dataset = pd.DataFrame(
        {'Company Name': companies_list,
         'Company Rating': ratings_list,
         'Position Title': positions_list,
         'Location' : locations_list,
         'Est. Salary' : salaries_list
        })
    
    glassdoor_dataset.to_csv(r'glassdoor_jobs_scraped.csv', index=False)

# 调用示例
glassdoor_scraper("https://www.glassdoor.com/index.htm")

内容的提问来源于stack exchange，提问作者Zoya Aqib