Python多线程Selenium在Digital Ocean服务器渲染超时问题

本地MacBook Air M2（8GB内存，macOS Sonoma 14.2.1）运行正常，但部署到Digital Ocean的1核1GB内存Ubuntu 23.10 Droplet的Django服务中，处理POST请求时抛出Timed out receiving message from renderer: 60.000错误。单线程运行无报错，但效率极低。

相关代码 #

主调度代码 #

from pathlib import Path
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool

from .scrapers.scraper_onu_odc import scrap_from_onu_odc
from .scrapers.scraper_peps import scrap_from_senaclaft_peps
from .scrapers.scraper_bcu import scrap_from_bcu_infractores_cheques
from .scrapers.scraper_onu_scs import scrap_from_onu_scs
from .scrapers.scraper_ofac import scrap_from_ofac
from .scrapers.scraper_google import scrap_from_google
from .scrapers.scraper_yahoo import scrap_from_yahoo
from .scrapers.scraper_bing import scrap_from_bing
from .scrapers.scraper_wikipedia import scrap_from_wikipedia
from .scrapers.scraper_fincen import scrap_from_fincen

BASE_DIR = Path(__file__).resolve().parent.parent.parent

def scrapping_pool(scraper):
    scraper[0](*scraper[1])


def get_search_results(info, uuid):

    paths = {
    'screenshots_path' : BASE_DIR / f'searcher/screenshots/{uuid}/',
    'certificate_path' : str(BASE_DIR / 'searcher/ssl_certificates/bcu.pem')
    }
    
    manager = Manager()
    results = manager.dict()

    person_to_search = info['person_to_search_name'] + ' ' + info['person_to_search_surname']

    params = (person_to_search, results, paths)
    scrapers = [
        (scrap_from_onu_odc, params),
        (scrap_from_senaclaft_peps, params),
        (scrap_from_bcu_infractores_cheques, params),
        (scrap_from_onu_scs, params),
        (scrap_from_ofac, params),
        (scrap_from_fincen, params),
        (scrap_from_google, params),
        (scrap_from_yahoo, params),
        (scrap_from_bing, params),
        (scrap_from_wikipedia, params)
    ]

    try:
        with ThreadPool(8) as pool:
            for result in pool.map(scrapping_pool, scrapers):
                print(result)
        pool.close()
    except Exception as e:
        print(e)

    return results, paths['screenshots_path']

示例爬虫代码 #

import os
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.chrome.service import Service

from ...pages import get_pages
from .utils import get_selenium_params

def scrap_from_yahoo(name, results_dic, paths):
    page_info = get_pages()
    screenshot_folder = paths['screenshots_path']
    selenium_params = get_selenium_params()

    browser = webdriver.Chrome(service=selenium_params['service'], options=selenium_params['options'])

    url = page_info['yahoo']['url'] + name.replace(' ', '+')
    
    os.makedirs(screenshot_folder, exist_ok=True)
    screenshot_path = f'{screenshot_folder}/yahoo.png'

    has_matches = False
    try:
        browser.get(url)
        browser.set_window_size(1366, 728)
        browser.save_screenshot(screenshot_path)
        has_matches = True
        
    except Exception as e:
        has_matches = False
        
    results_dic['yahoo'] = (has_matches, screenshot_path, screenshot_folder, url)

ChromeDriver参数配置 #

def get_selenium_params():
    driver_path = BASE_DIR / 'driver/chromedriver-linux64/chromedriver'
    driver_options = webdriver.ChromeOptions()
    driver_options.binary_location = "/usr/bin/chromium-browser"
    driver_options.add_argument('--headless')
    driver_options.add_argument('--start-maximized')
    driver_options.add_argument('--remote-debugging-pipe')
    driver_options.add_argument("enable-automation")
    driver_options.add_argument("--no-sandbox")
    driver_options.add_argument("--disable-extensions")
    driver_options.add_argument("--dns-prefetch-disable")
    driver_options.add_argument("--disable-gpu")
    driver_options.add_argument('--disable-dev-shm-usage')

    return {
        'service': Service(executable_path=driver_path),
        'options': driver_options
    }

错误信息 #

Message: timeout: Timed out receiving message from renderer: 60.000
Stacktrace:
#0 0x594000d8b8a3 <unknown>
#1 0x594000a818c6 <unknown>
#2 0x594000a697a0 <unknown>
#3 0x594000a67516 <unknown>
#4 0x594000a67b9f <unknown>
#5 0x594000ab53de <unknown>
#6 0x594000ab1d3d <unknown>
#7 0x594000afaaed <unknown>
#8 0x594000aee343 <unknown>
#9 0x594000abf593 <unknown>
#10 0x594000abff5e <unknown>
#11 0x594000d4f88b <unknown>
#12 0x594000d537e5 <unknown>
#13 0x594000d3d5b1 <unknown>
#14 0x594000d54372 <unknown>
#15 0x594000d228bf <unknown>
#16 0x594000d7a768 <unknown>
#17 0x594000d7a93b <unknown>
#18 0x594000d8a9f4 <unknown>
#19 0x707d49897b5a <unknown>

解决方案 #

1. 降低线程池规模 #

1核1GB内存的VPS无法同时承载8个Chrome实例（每个Chrome实例至少占用100-200MB内存），将线程数降至2-3：

with ThreadPool(2) as pool:

2. 优化Chrome参数减少资源占用 #

更新get_selenium_params，添加轻量配置：

def get_selenium_params():
    driver_path = BASE_DIR / 'driver/chromedriver-linux64/chromedriver'
    driver_options = webdriver.ChromeOptions()
    driver_options.binary_location = "/usr/bin/chromium-browser"
    driver_options.add_argument('--headless=new')  # 新版无头模式，资源占用更低
    driver_options.add_argument('--window-size=1024,768')  # 缩小渲染窗口
    driver_options.add_argument('--single-process')  # 强制单进程运行
    driver_options.add_argument("--no-sandbox")
    driver_options.add_argument("--disable-extensions")
    driver_options.add_argument("--dns-prefetch-disable")
    driver_options.add_argument("--disable-gpu")
    driver_options.add_argument('--disable-dev-shm-usage')
    driver_options.add_argument('--disable-images')  # 禁用图片加载（如果截图不需要）
    driver_options.add_argument('--blink-settings=imagesEnabled=false')

    return {
        'service': Service(executable_path=driver_path),
        'options': driver_options
    }

3. 添加强制超时并释放浏览器资源 #

每个爬虫函数必须显式关闭浏览器，避免内存泄漏，同时设置页面加载超时：

def scrap_from_yahoo(name, results_dic, paths):
    page_info = get_pages()
    screenshot_folder = paths['screenshots_path']
    selenium_params = get_selenium_params()

    browser = None
    try:
        browser = webdriver.Chrome(service=selenium_params['service'], options=selenium_params['options'])
        browser.set_page_load_timeout(30)  # 30秒页面加载超时
        browser.set_script_timeout(30)     # 30秒脚本执行超时
        
        url = page_info['yahoo']['url'] + name.replace(' ', '+')
        os.makedirs(screenshot_folder, exist_ok=True)
        screenshot_path = f'{screenshot_folder}/yahoo.png'

        browser.get(url)
        browser.set_window_size(1024, 768)
        browser.save_screenshot(screenshot_path)
        has_matches = True
        
    except Exception as e:
        has_matches = False
        print(f"Yahoo爬虫出错: {e}")
    finally:
        if browser:
            browser.quit()  # 强制关闭浏览器释放资源
        
    results_dic['yahoo'] = (has_matches, screenshot_path, screenshot_folder, url)

4. 补充Swap分区缓解内存压力 #

如果内存仍然不足，创建1GB Swap文件：

sudo fallocate -l 1G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
# 设置开机自动挂载
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab

5. 改用进程池替代线程池（可选） #

受Python GIL限制，线程池在Chrome渲染这类CPU密集任务上效率有限，改用multiprocessing.Pool：

from multiprocessing import Pool  # 替换ThreadPool
# ...
try:
    with Pool(2) as pool:
        for result in pool.map(scrapping_pool, scrapers):
            print(result)
    pool.close()
except Exception as e:
    print(e)

内容的提问来源于stack exchange，提问作者Gorosteaga