使用Beautiful Soup爬取hh.kz仅获取20个serp-item区块的问题

使用Python 3.9 + BeautifulSoup4解析almaty.hh.kz的职位页面时，仅能抓取到20个带serp-item类的div区块，但浏览器中实际显示40个。已尝试切换html5lib、html.parser、lxml解析器，以及用soup.select方法，结果均一致，推测剩余20个区块由JavaScript动态加载。

用户原代码：

import requests
import os
import time
import re
from bs4 import BeautifulSoup
import csv
import pandas as pd
df = pd.DataFrame({})
global_url = "https://almaty.hh.kz/"
headers = {
        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36"
    }
def get_all_pages():
    

    with open("data/page_1.html") as file:
        src = file.read()
#

    soup = BeautifulSoup(src,"lxml")
    #find("span", {"class":"pager-item-not-in-short-range"}).
    pages_count = int(soup.find("div",{"class":"pager"}).find_all("a")[-2].text)
    for i in range(1,pages_count+1):
        url = f"https://almaty.hh.kz/search/vacancy?area=160&clusters=true&enable_snippets=true&ored_clusters=true&professional_role=84&professional_role=116&professional_role=36&professional_role=157&professional_role=125&professional_role=156&professional_role=160&professional_role=10&professional_role=150&professional_role=25&professional_role=165&professional_role=73&professional_role=96&professional_role=164&professional_role=104&professional_role=112&professional_role=113&professional_role=148&professional_role=114&professional_role=121&professional_role=124&professional_role=20&search_period=30&hhtmFrom=vacancy_search_list&page={i}"
        r = requests.get(url = url,headers = headers)
        with open(f"data/page_{i}.html","w") as file:
            file.write(r.text)

        time.sleep(3)

    return pages_count+1


def collect_data(pages_count):
    for page in range(1, pages_count+1):
        with open(f"data/page_{page}.html") as file:
            src = file.read()

            soup = BeautifulSoup(src,"lxml")
            # item_cards = soup.find_all("div",{"class":"a-card__body ddl_product_link"})
            # print(len(item_cards))
            # for items in item_cards:
            #   product_title = items.find("a",{"class":"a-card__title link"}).text 
            #   product_price = items.find("span",{"class":"a-card__price-text"}).text
            #   product_geo = items.find("div",{"class":"a-card__subtitle"}).text
            #   print(f"Title:{product_title} - Price: {product_price} - GEO: {product_geo}")
            #items_divs = soup.find_all("div",{"class":"serp-item"})
            items_divs = soup.find_all("div",{"class":"serp-item"})
            print(len(items_divs))
            urls =[]
            for item in items_divs:
                item_url = item.find("span",{"data-page-analytics-event":"vacancy_search_suitable_item"}).find("a",{"class":"serp-item__title"}).get("href")
                urls.append(item_url)
            with open("items_urls.txt","w") as file:
                for url in urls:
                    file.write(f"{url}\n")
            get_data(file_path="items_urls.txt")

def get_data(file_path):
    result_list = []
    with open(file_path) as file:
        urls_list = file.readlines()
        clear_urls_list =[]
        for url in urls_list:
            url = url.strip()
            clear_urls_list.append(url)
    
    i=0
    for url in clear_urls_list:
        i+=1
        response = requests.get(url=url,headers=headers)
        soup = BeautifulSoup(response.text,"lxml")


        try:
            item_name = soup.find("div",{"class":"main-content"}).find("h1",{"data-qa":"vacancy-title"}).text.strip()
        except:
            item_name = 'E1'

        try:
            item_salary = soup.find("div",{"class":"main-content"}).find("div",{"data-qa":"vacancy-salary"}).text.strip()
        except:
            item_salary = 'E2'

        try:
            item_exp = soup.find("div",{"class":"main-content"}).find("span",{"data-qa":"vacancy-experience"}).text.strip()
        except:
            item_exp = 'E3'

        try:
            company_name = soup.find("div",{"class":"main-content"}).find("span",{"class":"vacancy-company-name"}).find("span").text.strip()
        except:
            company_name = 'E4'

        try:
            if soup.find("div",{"class":"main-content"}).find("p",{"class":"vacancy-creation-time-redesigned"}):
                date = soup.find("div",{"class":"main-content"}).find("p",{"class":"vacancy-creation-time-redesigned"}).text.strip()
            else:
                date = soup.find("div",{"class":"main-content"}).find("p",{"class":"vacancy-creation-time"}).text.strip()
        except:
            date = 'E5'

        try:
            if soup.find("div",{"class":"main-content"}).find("span",{"data-qa":"vacancy-view-raw-address"}):
                address = soup.find("div",{"class":"main-content"}).find("span",{"data-qa":"vacancy-view-raw-address"}).text
            elif soup.find("div",{"class":"main-content"}).find("div",{"class":"vacancy-company-bottom"}).find("p", {"data-qa":"vacancy-view-location"}):
                address = soup.find("div",{"class":"main-content"}).find("div",{"class":"vacancy-company-bottom"}).find("p", {"data-qa":"vacancy-view-location"}).text
            elif soup.find("div",{"class":"main-content"}).find("div",{"class":"block-employer--jHuyqacEkkrEkSl3Yg3M"}):
                address = soup.find("div",{"class":"main-content"}).find("div",{"class":"block-employer--jHuyqacEkkrEkSl3Yg3M"}).find("p", {"data-qa":"vacancy-view-location"}).text
        except:
            address = 'Алматы'

        try:
            zanyatost = soup.find("div",{"class":"main-content"}).find("p",{"data-qa":"vacancy-view-employment-mode"}).find("span").text.strip()
        except:
            zanyatost = 'E7'

        try:
            zanyatost2 = soup.find("div",{"class":"main-content"}).find("p",{"data-qa":"vacancy-view-employment-mode"}).text.lstrip(', ')
        except:
            zanyatost2 = 'E8'
        print(i)

        with open('test.csv','a',encoding ="utf-8") as file:
            writer = csv.writer(file)
            writer.writerow(
                (
                    item_name,
                    item_salary,
                    item_exp,
                    company_name,
                    date,
                    address,
                    zanyatost,
                    zanyatost2
                )
            )


def main():
    with open('test.csv','w',encoding ="utf-8") as file:
            writer = csv.writer(file)
            writer.writerow(
                (
                    'Должность',
                    "Зарплата",
                    "Опыт",
                    "Компания",
                    "Дата обьявления",
                    "Район",
                    "Тип занятости",
                    "Тип занятости2"
                )
            )
    pages_count = get_all_pages()
    #print(pages_count)

    collect_data(pages_count=pages_count)
    # #get_data(file_path="items_urls.txt")
    # df.to_excel('./test.xlsx')

if __name__ == '__main__':
    main()

方法一：用Selenium模拟浏览器渲染 #

Selenium能模拟真实浏览器加载页面，等待JavaScript执行完毕后再抓取完整内容，步骤如下：

1. 安装依赖：

pip install selenium webdriver-manager

2. 替换原get_all_pages函数：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def get_all_pages():
    # 配置浏览器选项
    options = webdriver.ChromeOptions()
    options.add_argument("--headless=new")  # 无头模式，不显示浏览器窗口
    options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

    # 加载第一页并计算总页数
    first_url = "https://almaty.hh.kz/search/vacancy?area=160&clusters=true&enable_snippets=true&ored_clusters=true&professional_role=84&professional_role=116&professional_role=36&professional_role=157&professional_role=125&professional_role=156&professional_role=160&professional_role=10&professional_role=150&professional_role=25&professional_role=165&professional_role=73&professional_role=96&professional_role=164&professional_role=104&professional_role=112&professional_role=113&professional_role=148&professional_role=114&professional_role=121&professional_role=124&professional_role=20&search_period=30&hhtmFrom=vacancy_search_list&page=0"
    driver.get(first_url)
    # 等待页面加载完成，直到目标元素出现
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "serp-item")))
    
    # 解析总页数
    soup = BeautifulSoup(driver.page_source, "lxml")
    pages_count = int(soup.find("div", {"class":"pager"}).find_all("a")[-2].text)
    
    # 保存第一页内容
    with open("data/page_1.html", "w", encoding="utf-8") as file:
        file.write(driver.page_source)
    
    # 遍历其他页面
    for i in range(1, pages_count+1):
        url = f"https://almaty.hh.kz/search/vacancy?area=160&clusters=true&enable_snippets=true&ored_clusters=true&professional_role=84&professional_role=116&professional_role=36&professional_role=157&professional_role=125&professional_role=156&professional_role=160&professional_role=10&professional_role=150&professional_role=25&professional_role=165&professional_role=73&professional_role=96&professional_role=164&professional_role=104&professional_role=112&professional_role=113&professional_role=148&professional_role=114&professional_role=121&professional_role=124&professional_role=20&search_period=30&hhtmFrom=vacancy_search_list&page={i}"
        driver.get(url)
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "serp-item")))
        time.sleep(2)  # 额外等待确保JS加载完成
        
        with open(f"data/page_{i+1}.html", "w", encoding="utf-8") as file:
            file.write(driver.page_source)
    
    driver.quit()
    return pages_count + 1

方法二：直接请求API接口（更高效） #

HH的职位数据通常通过AJAX接口返回JSON格式内容，无需渲染页面，速度更快更稳定：

1. 示例代码：

import requests

headers = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36",
    "accept": "application/json"
}

def get_vacancies_from_api():
    params = {
        "area": "160",  # 阿拉木图区域ID
        "professional_role": ["84", "116", "36", "157", "125", "156", "160", "10", "150", "25", "165", "73", "96", "164", "104", "112", "113", "148", "114", "121", "124", "20"],
        "search_period": "30",
        "per_page": "40",  # 每页请求40条数据
        "page": 0  # 页码从0开始
    }
    url = "https://api.hh.ru/vacancies"
    response = requests.get(url, headers=headers, params=params)
    data = response.json()
    
    total_pages = data["pages"]
    urls = [item["alternate_url"] for item in data["items"]]
    
    # 遍历所有页码
    for page in range(1, total_pages):
        params["page"] = page
        response = requests.get(url, headers=headers, params=params)
        data = response.json()
        urls.extend([item["alternate_url"] for item in data["items"]])
        time.sleep(2)  # 避免请求过于频繁
    
    # 保存职位URL到文件
    with open("items_urls.txt", "w", encoding="utf-8") as file:
        for url in urls:
            file.write(f"{url}\n")
    
    return urls

之后可以直接调用get_vacancies_from_api()获取所有职位URL，再传入原get_data函数处理即可。

内容的提问来源于stack exchange，提问作者Trixnez