使用Python+BeautifulSoup/Selenium爬OilPrice新闻过慢，求提速方案

以下Python代码通过Selenium和BeautifulSoup爬取OilPrice网站原油板块新闻的标题与发布日期，功能正常但多页面循环耗时过长，需要优化以爬取更多页面：

from selenium import webdriver
from bs4 import BeautifulSoup
import time
import requests
import os
title_list = []
date_list = []
os.chdir(r'C:\Users\Foued Azuz 14\mypythonfiles')
driver = webdriver.Chrome()
for page in range (104,110):
    url = f"https://oilprice.com/Energy/Crude-Oil/Page-{page}.html"
    driver.get(url)
    from bs4 import BeautifulSoup
    html = driver.page_source
    soup = BeautifulSoup(html, 'lxml')
    title= soup.find_all('div',class_='categoryArticle__content')
    for i in title:
            d = i.find('h2', class_='categoryArticle__title').text
            h = i.find('p', class_='categoryArticle__meta')
            date_str = h.get_text()
            date = date_str.split(' at ')[0]
            if 'oil' in d.lower():
                title_list.append(d)
                date_list.append(date)

data = {'date': date_list, 'Title': title_list}
scrapdataoil = pd.DataFrame(data)
scrapdataoil.to_csv('scrapdataoil104110.csv', index=False)

优化方案 #

1. 替换Selenium为直接请求（优先推荐） #

Selenium需要加载完整浏览器渲染页面，速度远慢于直接用requests请求HTML。若目标网站无强反爬机制，直接改用requests：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import os

title_list = []
date_list = []
os.chdir(r'C:\Users\Foued Azuz 14\mypythonfiles')
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
}

for page in range(104, 110):
    url = f"https://oilprice.com/Energy/Crude-Oil/Page-{page}.html"
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'lxml')
    articles = soup.find_all('div', class_='categoryArticle__content')
    for article in articles:
        title = article.find('h2', class_='categoryArticle__title').text.strip()
        meta = article.find('p', class_='categoryArticle__meta').get_text()
        date = meta.split(' at ')[0]
        if 'oil' in title.lower():
            title_list.append(title)
            date_list.append(date)

data = {'date': date_list, 'Title': title_list}
scrapdataoil = pd.DataFrame(data)
scrapdataoil.to_csv('scrapdataoil104110.csv', index=False)

优化点：添加User-Agent模拟浏览器请求；移除循环内重复的模块导入；用strip()清理文本冗余空格；补充原代码缺失的pandas导入。

2. 启用Selenium无头模式（必须用Selenium时） #

如果页面依赖JS动态渲染无法直接请求，启用无头模式减少浏览器渲染开销：

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
import pandas as pd
import os

title_list = []
date_list = []
os.chdir(r'C:\Users\Foued Azuz 14\mypythonfiles')

# 配置无头模式
chrome_options = Options()
chrome_options.add_argument("--headless=new")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--no-sandbox")
driver = webdriver.Chrome(options=chrome_options)

for page in range(104, 110):
    url = f"https://oilprice.com/Energy/Crude-Oil/Page-{page}.html"
    driver.get(url)
    soup = BeautifulSoup(driver.page_source, 'lxml')
    articles = soup.find_all('div', class_='categoryArticle__content')
    for article in articles:
        title = article.find('h2', class_='categoryArticle__title').text.strip()
        meta = article.find('p', class_='categoryArticle__meta').get_text()
        date = meta.split(' at ')[0]
        if 'oil' in title.lower():
            title_list.append(title)
            date_list.append(date)

driver.quit()  # 释放浏览器资源
data = {'date': date_list, 'Title': title_list}
scrapdataoil = pd.DataFrame(data)
scrapdataoil.to_csv('scrapdataoil104110.csv', index=False)

优化点：无头模式不显示浏览器窗口，大幅降低资源占用；循环结束后调用driver.quit()避免资源泄漏。

3. 异步批量请求（大量页面爬取） #

使用aiohttp实现异步请求，同时获取多个页面内容，进一步压缩总耗时：

import aiohttp
import asyncio
from bs4 import BeautifulSoup
import pandas as pd
import os

title_list = []
date_list = []
os.chdir(r'C:\Users\Foued Azuz 14\mypythonfiles')
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
}

async def fetch_page(session, page_num):
    url = f"https://oilprice.com/Energy/Crude-Oil/Page-{page_num}.html"
    async with session.get(url, headers=headers) as response:
        html = await response.text()
        soup = BeautifulSoup(html, 'lxml')
        articles = soup.find_all('div', class_='categoryArticle__content')
        for article in articles:
            title = article.find('h2', class_='categoryArticle__title').text.strip()
            meta = article.find('p', class_='categoryArticle__meta').get_text()
            date = meta.split(' at ')[0]
            if 'oil' in title.lower():
                title_list.append(title)
                date_list.append(date)

async def main():
    async with aiohttp.ClientSession() as session:
        tasks = [fetch_page(session, page) for page in range(104, 110)]
        await asyncio.gather(*tasks)

if __name__ == "__main__":
    asyncio.run(main())
    data = {'date': date_list, 'Title': title_list}
    scrapdataoil = pd.DataFrame(data)
    scrapdataoil.to_csv('scrapdataoil104110.csv', index=False)

优化点：异步请求可同时处理多个页面，避免等待单个请求响应的时间，适合爬取数百上千页的场景。

内容的提问来源于stack exchange，提问作者Foha2001