Selenium多进程打开浏览器闪退及SSL握手失败问题求助

使用multiprocessing.Pool打开多个浏览器窗口时，窗口会瞬间关闭，推测由以下SSL握手失败错误导致：

[4972:10204:0215/233710.867:ERROR:ssl_client_socket_impl.cc(974)] handshake failed; returned -1, SSL error code 1, net_error -101

但仅打开单个页面时，虽存在该错误仍能正常运行。相关代码如下：

import requests
from bs4 import BeautifulSoup
import time
import os
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from multiprocessing import Pool


url = 'https://www.pik.ru/search/storehouse'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0'
}


def download_pages_objects(url):
    if os.path.isfile(r'C:\Users\kraz1\OneDrive\Рабочий стол\Антон\python\парсинг\кладовочная\pik_links.txt') == True:
        os.remove(
            r'C:\Users\kraz1\OneDrive\Рабочий стол\Антон\python\парсинг\кладовочная\pik_links.txt')

    list_links = []
    req = requests.get(url, headers=headers)
    soup = BeautifulSoup(req.text, "html5lib")

    for i in soup.find_all("a", class_="styles__ProjectCard-uyo9w7-0 friPgx"):
        list_links.append('https://www.pik.ru'+i.get('href')+'\n')

    list_links = list(set(list_links))

    with open(r'C:\Users\kraz1\OneDrive\Рабочий стол\Антон\python\парсинг\кладовочная\pik_links.txt', 'a') as file:
        for link in list_links:
            file.write(link)


def get_list_objects_links(url):
    download_pages_objects(url)

    list_of_links = []
    with open(r'C:\Users\kraz1\OneDrive\Рабочий стол\Антон\python\парсинг\кладовочная\pik_links.txt', 'r') as file:
        for item in file:
            list_of_links.append(item[:-1])

    return list_of_links


def operation(url):

    # options = webdriver.ChromeOptions()
    # options.add_argument('--ignore-certificate-errors-spki-list')
    # options.add_argument('--ignore-ssl-errors')
    # options.add_argument('--ignore-certificate-error')

    driver = webdriver.Chrome()

    driver.get(url)
    time.sleep(40)

    pahe = driver.find_elements(By.CSS_SELECTOR, '.sc-gsnTZi.fWJuXR')

    count = len(pahe)
    print(len(pahe))
    while count != 0:
        count = 0

        for item in pahe:
            if ('Показать' in item.text):
                actions = ActionChains(driver)
                actions.move_to_element(item).perform()
                print(item.text, '  ', item, '\n\n')
                time.sleep(2)
                item.click()
                time.sleep(2)
                count += 1

        pahe = driver.find_elements(By.CSS_SELECTOR, '.sc-gsnTZi.fWJuXR')
        print(len(pahe))

    response = driver.page_source

    list_refer_storehouse_on_page = []
    soup = BeautifulSoup(response, 'lxml')
    for item in (soup.find_all('div', class_='sc-htiqpR fhmJpy')):
        list_refer_storehouse_on_page.append(
            'https://www.pik.ru' + item.find('a').get('href'))
    print(list_refer_storehouse_on_page)

    driver.close()
    driver.quit()

    with open(fr'C:\Users\kraz1\OneDrive\Рабочий стол\Антон\python\парсинг\кладовочная\data_pages\{url.split('/')[-2]}.txt', 'w') as file:
        for link in list_refer_storehouse_on_page:
            file.write(f'{link}\n')


def main():
    list_links = get_list_objects_links(url)
    p = Pool(processes=1)
    # p.map(operation, list_links[0])
    operation(list_links[0])


if __name__ == '__main__':
    main()

注释掉的SSL相关配置：

# options = webdriver.ChromeOptions()
# options.add_argument('--ignore-certificate-errors-spki-list')
# options.add_argument('--ignore-ssl-errors')
# options.add_argument('--ignore-certificate-error')

1. 启用SSL证书忽略选项 #

取消注释SSL相关配置并修正拼写错误，将配置应用到WebDriver初始化中，解决握手失败问题：

def operation(url):
    options = webdriver.ChromeOptions()
    options.add_argument('--ignore-certificate-errors-spki-list')
    options.add_argument('--ignore-ssl-errors')
    options.add_argument('--ignore-certificate-errors')
    options.add_argument('--start-maximized')  # 可选，避免窗口异常最小化
    driver = webdriver.Chrome(options=options)
    # 后续代码保持不变

2. 修复进程池调用逻辑 #

当前代码中p.map(operation, list_links[0])存在错误，map需要传入可迭代对象，同时用with语句管理进程池确保资源正确释放：

def main():
    list_links = get_list_objects_links(url)
    with Pool(processes=3) as p:  # 根据机器性能调整进程数，建议不超过CPU核心数
        p.map(operation, list_links)

3. 添加异常捕获与资源清理 #

多进程环境下子进程错误不会直接输出，添加异常捕获定位问题并确保出错时关闭浏览器：

def operation(url):
    try:
        options = webdriver.ChromeOptions()
        options.add_argument('--ignore-certificate-errors-spki-list')
        options.add_argument('--ignore-ssl-errors')
        options.add_argument('--ignore-certificate-errors')
        driver = webdriver.Chrome(options=options)
        
        driver.get(url)
        time.sleep(40)
        
        # 原有页面操作代码
        
        driver.close()
        driver.quit()
        
        # 写入文件代码
    except Exception as e:
        print(f"处理URL {url} 出错: {str(e)}")
        if 'driver' in locals():
            driver.quit()

4. 替换固定等待为显式等待 #

time.sleep(40)效率低且不稳定，改用Selenium显式等待确保元素加载完成：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 替换time.sleep(40)
wait = WebDriverWait(driver, 40)
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.sc-gsnTZi.fWJuXR')))

内容的提问来源于stack exchange，提问作者POPIF