进程间共享Selenium Driver报错：无法序列化本地对象

问题场景 #

尝试通过multiprocessing.Manager()和Queue在两个子进程间共享Chrome Driver对象，一个进程负责初始化Driver并刷新页面保活，另一个进程检测选课空位并执行选课操作，运行时报错无法pickle本地对象。

简化测试代码 #

from selenium.webdriver import Chrome
from multiprocessing import Queue, Manager, Process
import time

def find(q):
    driver = Chrome()
    driver.get('https://querycourse.ntust.edu.tw/querycourse/api/courses')
    q.put(driver)

def refresh(q):
    driver = q.get()
    while True:
        time.sleep(9)
        driver.refresh()

if __name__=='__main__':
    with Manager() as manager:
        q = manager.Queue()
        p1 = Process(target=find, args=(q,))
        p2 = Process(target=refresh, args=(q,))
        
        p1.start()
        time.sleep(3)
        p2.start()

        p1.join()
        p2.join()

报错信息 #

Process Process-2:
Traceback (most recent call last):
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 314, in _bootstrap
    self.run()
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "D:\python_training\web crawler\kk_manager_queue.py", line 8, in find
    q.put(driver)
  File "<string>", line 2, in put
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\managers.py", line 817, in _callmethod
    conn.send((self._id, methodname, args, kwds))
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\connection.py", line 211, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object '_createenviron.<locals>.encodekey'

错误原因 #

1. Chrome Driver无法被序列化：multiprocessing在进程间传递对象时，需要将对象序列化为字节流（pickle），但Chrome Driver内部包含大量非序列化资源（如浏览器进程句柄、本地环境变量的闭包引用），这些内容无法被pickle处理，直接传递会触发报错。
2. Driver本身不支持跨进程共享：即使解决了序列化问题，Chrome Driver也不是进程安全的，多进程同时操作同一个Driver会导致页面状态混乱、操作失效等不可预期的问题。

解决方案 #

针对选课需求（保活刷新+空位检测选课），不需要跨进程共享Driver，推荐以下两种可行方案：

方案1：单进程+多线程（简单高效） #

使用线程替代进程，通过线程锁保证同一时间只有一个线程操作Driver，避免并发冲突。

修改后的代码：

import urllib.request as req
import json
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.wait import WebDriverWait
from threading import Thread, Lock

# 全局锁，控制Driver的访问
driver_lock = Lock()
driver = None

def find(url, requestData, chooseStudent):
    global driver
    while True:
        # 发起API请求检测空位
        request=req.Request(url, headers={
            "content-type":"application/json; charset=utf-8"
        }, data=json.dumps(requestData).encode("utf-8"))

        with req.urlopen(request) as response:
            result=response.read().decode("utf-8")

        result=json.loads(result)

        if int(result[0]["ChooseStudent"]) < chooseStudent:
            # 获得锁后操作Driver
            with driver_lock:
                ADDgo = driver.find_element(By.ID, "SingleAdd")
                ADDgo.click()
                print("已执行选课操作")
            break
        time.sleep(2)  # 控制检测频率，避免请求过于频繁

def refresh():
    global driver
    account='XXXXXXXX'
    password='XXXXXXXX'

    # 初始化Driver并登录
    driver = Chrome()
    driver.get("https://stuinfosys.ntust.edu.tw/NTUSTSSOServ/SSO/Login/CourseSelection")
    UserName = WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.NAME,"UserName"))
    UserName.send_keys(account)
    Password = driver.find_element(By.NAME, "Password")
    Password.send_keys(password)
    btnLogIn = driver.find_element(By.NAME, "btnLogIn")
    btnLogIn.click()
    time.sleep(3)  # 等待登录完成

    # 循环刷新页面保活
    while True:
        with driver_lock:
            driver.refresh()
        time.sleep(10)

if __name__=='__main__':
    url="https://querycourse.ntust.edu.tw/querycourse/api/courses"
    requestData={"Semester":"1112","CourseNo":"GE3710302","CourseName":"","CourseTeacher":"","Dimension":"","CourseNotes":"","ForeignLanguage":0,"OnlyGeneral":0,"OnleyNTUST":0,"OnlyMaster":0,"OnlyUnderGraduate":0,"OnlyNode":0,"Language":"zh"}
    chooseStudent=51

    # 创建线程
    t_refresh = Thread(target=refresh)
    t_find = Thread(target=find, args=(url, requestData, chooseStudent))

    t_refresh.start()
    time.sleep(5)  # 等待登录完成再启动检测线程
    t_find.start()

    t_find.join()
    t_refresh.join()

方案2：进程间传递指令（适合复杂场景） #

让一个进程持有Driver，另一个进程通过Queue发送操作指令（如"选课"），持有Driver的进程接收指令并执行对应的操作。这种方式避免了共享Driver对象，而是传递操作命令。

示例代码框架：

from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
import time
from multiprocessing import Queue, Process
import urllib.request as req
import json

def find(url, requestData, chooseStudent, cmd_queue):
    while True:
        request=req.Request(url, headers={
            "content-type":"application/json; charset=utf-8"
        }, data=json.dumps(requestData).encode("utf-8"))

        with req.urlopen(request) as response:
            result=response.read().decode("utf-8")

        result=json.loads(result)

        if int(result[0]["ChooseStudent"]) < chooseStudent:
            cmd_queue.put("SELECT")  # 发送选课指令
            break
        time.sleep(2)

def refresh(cmd_queue):
    account='XXXXXXXX'
    password='XXXXXXXX'

    driver = Chrome()
    driver.get("https://stuinfosys.ntust.edu.tw/NTUSTSSOServ/SSO/Login/CourseSelection")
    UserName = WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.NAME,"UserName"))
    UserName.send_keys(account)
    Password = driver.find_element(By.NAME, "Password")
    Password.send_keys(password)
    btnLogIn = driver.find_element(By.NAME, "btnLogIn")
    btnLogIn.click()
    time.sleep(3)

    while True:
        # 先检查是否有指令需要执行
        if not cmd_queue.empty():
            cmd = cmd_queue.get()
            if cmd == "SELECT":
                ADDgo = driver.find_element(By.ID, "SingleAdd")
                ADDgo.click()
                print("已执行选课操作")
                break  # 完成选课后退出循环
        # 刷新页面
        driver.refresh()
        time.sleep(10)

if __name__=='__main__':
    url="https://querycourse.ntust.edu.tw/querycourse/api/courses"
    requestData={"Semester":"1112","CourseNo":"GE3710302","CourseName":"","CourseTeacher":"","Dimension":"","CourseNotes":"","ForeignLanguage":0,"OnlyGeneral":0,"OnleyNTUST":0,"OnlyMaster":0,"OnlyUnderGraduate":0,"OnlyNode":0,"Language":"zh"}
    chooseStudent=51

    cmd_queue = Queue()
    p_find = Process(target=find, args=(url, requestData, chooseStudent, cmd_queue))
    p_refresh = Process(target=refresh, args=(cmd_queue,))

    p_refresh.start()
    time.sleep(5)
    p_find.start()

    p_find.join()
    p_refresh.join()

内容的提问来源于stack exchange，提问作者Nicholas