如何用Python Selenium实现多关键词搜索与网页爬取？

需求说明 #

需要用Selenium实现网页爬取，核心流程如下：

• 依次在网站搜索框中搜索指定关键词（无法通过布尔运算符批量搜索，需逐个处理）
• 对每个关键词的搜索结果页，点击所有文章链接爬取完整内容
• 完成单个关键词的所有文章爬取后，返回搜索页面，继续处理下一个关键词

现有爬虫代码已能在部分网站运行，但缺少多关键词循环搜索功能，需更新代码。

修改后的完整代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

# -------------------------- 配置区域 --------------------------
# 替换为你的目标网站首页URL
HOME_URL = "https://www.salute.gov.it/portale/home.html"
# 替换为需要搜索的关键词列表
SEARCH_KEYWORDS = ["big data", "人工智能", "公共卫生"]
# 替换为实际页面元素的XPATH（根据目标网站调整）
XPATHS = {
    "cookie_banner": "",  # 同意cookie按钮的XPATH
    "search_box": "",     # 搜索框的XPATH
    "search_submit": "",  # 搜索提交按钮的XPATH（如果需要点击提交）
    "article_links": "",  # 搜索结果中文章链接的XPATH
    "article_title": "",  # 文章详情页标题的XPATH
    "article_date": "",   # 文章详情页发布日期的XPATH
    "article_content": "",# 文章详情页内容的XPATH
    "next_page": ""       # 搜索结果下一页按钮的XPATH
}
# -------------------------------------------------------------

def accept_cookies(driver):
    """处理Cookie弹窗"""
    try:
        cookie_btn = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, XPATHS["cookie_banner"]))
        )
        cookie_btn.click()
    except Exception as e:
        print(f"未检测到Cookie弹窗或处理失败: {str(e)}")

def search_keyword(driver, keyword):
    """在搜索框中输入关键词并执行搜索"""
    try:
        # 回到首页确保搜索框可访问
        driver.get(HOME_URL)
        accept_cookies(driver)
        
        # 等待搜索框加载并输入关键词
        search_box = WebDriverWait(driver, 10).until(
            EC.visibility_of_element_located((By.XPATH, XPATHS["search_box"]))
        )
        search_box.clear()
        search_box.send_keys(keyword)
        # 按回车执行搜索，若需点击提交按钮可替换为下方注释代码
        search_box.send_keys(Keys.ENTER)
        # search_btn = WebDriverWait(driver, 10).until(
        #     EC.element_to_be_clickable((By.XPATH, XPATHS["search_submit"]))
        # )
        # search_btn.click()
        
        print(f"已执行关键词搜索: {keyword}")
    except Exception as e:
        print(f"搜索关键词 {keyword} 失败: {str(e)}")
        raise

def scrape_article(driver, article_url):
    """爬取单篇文章的详情内容"""
    try:
        driver.get(article_url)
        title = WebDriverWait(driver, 10).until(
            EC.visibility_of_element_located((By.XPATH, XPATHS["article_title"]))
        ).text
        date = WebDriverWait(driver, 10).until(
            EC.visibility_of_element_located((By.XPATH, XPATHS["article_date"]))
        ).text
        content = WebDriverWait(driver, 10).until(
            EC.visibility_of_element_located((By.XPATH, XPATHS["article_content"]))
        ).text
        return {
            "关键词": driver.current_keyword,  # 标记当前文章所属关键词
            "标题": title,
            "日期": date,
            "URL": article_url,
            "内容": content
        }
    except Exception as e:
        print(f"爬取文章 {article_url} 失败: {str(e)}")
        return None

def scrape_search_results(driver):
    """爬取当前搜索结果页的所有文章，并处理分页"""
    all_articles = []
    current_page_url = driver.current_url
    
    while True:
        print(f"正在爬取搜索结果页: {current_page_url}")
        try:
            # 等待文章链接加载完成
            article_elements = WebDriverWait(driver, 10).until(
                EC.presence_of_all_elements_located((By.XPATH, XPATHS["article_links"]))
            )
            # 获取所有有效文章URL
            article_urls = [elem.get_attribute("href") for elem in article_elements if elem.get_attribute("href")]
            
            # 逐个爬取文章
            for url in article_urls:
                article_data = scrape_article(driver, url)
                if article_data:
                    all_articles.append(article_data)
            
            # 尝试获取下一页
            try:
                next_btn = WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, XPATHS["next_page"]))
                )
                next_btn.click()
                # 等待页面跳转完成
                WebDriverWait(driver, 10).until(EC.url_changes(current_page_url))
                current_page_url = driver.current_url
            except Exception as e:
                print("已爬完所有搜索结果页")
                break
                
        except Exception as e:
            print(f"处理搜索结果页失败: {str(e)}")
            break
    
    return all_articles

if __name__ == "__main__":
    # 初始化浏览器驱动
    driver = webdriver.Chrome()
    driver.maximize_window()
    all_data = []
    
    try:
        # 循环处理每个关键词
        for keyword in SEARCH_KEYWORDS:
            driver.current_keyword = keyword  # 标记当前处理的关键词
            search_keyword(driver, keyword)
            keyword_articles = scrape_search_results(driver)
            all_data.extend(keyword_articles)
            print(f"关键词 {keyword} 爬取完成，共获取 {len(keyword_articles)} 篇文章")
        
        # 保存数据到Excel
        if all_data:
            df = pd.DataFrame(all_data)
            print(df.head())
            df.to_excel("多关键词爬取结果.xlsx", index=False, encoding="utf-8-sig")
            print("数据已成功保存到 多关键词爬取结果.xlsx")
        else:
            print("未爬取到任何数据")
            
    except Exception as e:
        print(f"爬虫执行出错: {str(e)}")
    finally:
        # 关闭浏览器
        driver.quit()

关键修改说明 #

1. 新增多关键词循环逻辑：添加SEARCH_KEYWORDS列表，通过循环逐个处理每个关键词
2. 拆分功能函数：将Cookie处理、关键词搜索、文章爬取、结果分页处理拆分为独立函数，代码更易维护
3. 替换隐式等待为显式等待：使用WebDriverWait替代implicitly_wait，提升元素定位的可靠性
4. 添加关键词标记：在文章数据中标记所属关键词，方便后续数据分析
5. 完善错误处理：每个功能模块都添加了异常捕获，避免单个环节出错导致整个爬虫崩溃
6. 配置化管理XPATH：将所有页面元素的XPATH集中到配置字典，方便根据目标网站调整

使用注意事项 #

• 必须根据目标网站的实际HTML结构，替换XPATHS字典中的所有空XPATH值
• 如果搜索需要点击提交按钮而非回车，注释掉回车代码，启用提交按钮点击的代码
• 若目标网站有反爬机制，需添加适当的延时（可使用time.sleep()）或代理IP

内容的提问来源于stack exchange，提问作者Roberto Artiaco