Celery任务中运行无头Firefox时WebDriver意外关闭报错

问题详情 #

在Flask应用中通过Celery任务以无头模式运行Selenium Firefox时，触发错误：

selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 127

相关代码 #

from selenium.webdriver.firefox.service import Service
import sys, os
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep
from dotenv import load_dotenv
from check_email import EmailedReferrals, Referrals
from sqlalchemy import create_engine, Column, Integer, String, Date, Text
from sqlalchemy.orm import declarative_base 
from sqlalchemy.orm import sessionmaker
from datetime import datetime
import logging

logger = logging.getLogger('celery')
logger.setLevel(logging.DEBUG)

load_dotenv()

env_conf = os.getenv("CONFIG")
Base = declarative_base()

if env_conf == "dev":    
    driver = webdriver.Firefox()
    DATABASE_URL = os.getenv('DEV_DATABASE_URL')    
else: 
    # Set up Firefox options
    options = Options()
    # Set window size
    options.add_argument("--window-size=1920,1080")
    options.add_argument("-headless")
    # Include the path to the Firefox binary
    firefox_binary_path = '/usr/bin/firefox'
    geckodriver_path = '/home/fusion/bin/geckodriver'
    options.binary_location = firefox_binary_path
    service = Service(executable_path=geckodriver_path)
    driver = webdriver.Firefox(service=service, options=options)
    DATABASE_URL = os.getenv('PROD_DATABASE_URL')
engine = create_engine(DATABASE_URL)

Base.metadata.create_all(engine)

Session = sessionmaker(bind=engine)
session = Session()

def get_row_xpath(row, cell): #pulls xpath of rows and cells from Incoming Referrals View
    ctl = str(row).zfill(2)
    action = row - 3
    id_cell = f"""/html/body/form/table/tbody/tr[4]/td[2]/table/tbody/tr/td/table/tbody/tr[2]/td/div/table/tbody\
        /tr/td[3]/div/table/tbody/tr[2]/td/div/table/tbody/tr[{row}]/td[{cell}]"""
    select_element = f"""//*[@id="ucViewGrid_dgView_ctl{ctl}_Actions_{action}_0_0_ActionItems"]"""
    go_button = f"""#ucViewGrid_dgView_ctl{ctl}_Actions_{action}_0_0_ActionButton > input:nth-child(1)"""
    return id_cell, select_element, go_button

def main(referral_id_from_flask, comment):
    print('hello from main')
    logger.info("hello from main")
    driver.get("https://www.example.com")
    #Click Account and sign in
    driver.find_element(By.XPATH, '//*[@id="UserNameTextBox"]').send_keys(os.getenv('USERNAME'))
    driver.find_element(By.XPATH, '//*[@id="PasswordTextBox"]').send_keys(os.getenv('PASSWORD'))
    driver.find_element(By.XPATH, '//*[@id="btnLoginHack"]').click()
    sleep(2)
    driver.find_element(By.XPATH, '//*[@id="ctl13"]').click()
    #loop through table
    cell = 3
    for row in range(3, 52):
        logger.info(f"Looping: {row}")
        print(f"Loop #: {row}")
        id_cell = driver.find_element(By.XPATH, get_row_xpath(row, cell)[0]).text
        referral_id = id_cell.split('\n')[0].strip()
        #find row with the correct
        if referral_id == referral_id_from_flask:
            #Select View Online Referral
            select_element = driver.find_element(By.XPATH, get_row_xpath(row, cell)[1])
            select = Select(select_element)
            select.select_by_visible_text("View")
            #click go
            driver.find_element(By.CSS_SELECTOR, get_row_xpath(row, cell)[2]).click()
            #we are in View Online Referral of the patient
            #Find select box
            #select value 0  which is accept patient
            select_element = driver.find_element(By.XPATH, """//*[@id="dgProviders_ctl02_ddResponse"]""")
            select = Select(select_element)
            select.select_by_value("1")
            sleep(0.5)
            #comment box
            # wait up to 10 seconds for the element to be present and clickable
            comment_box = WebDriverWait(driver, 10).until(
                EC.element_to_be_clickable((By.XPATH, '//*[@id="dgProviders_ctl02_txtProviderComments"]'))
            )
            comment_box.clear()
            sleep(0.5) # Add a small delay here
            comment_box.send_keys(comment)
            sleep(0.5) # Add a small delay here
            #click send response
            #driver.find_element(By.XPATH, """//*[@id="ButtonBarSendResponse"]""").click()
            #add to query
            r = session.query(Referrals).filter(Referrals.referral_id == int(referral_id)).first()
            r.referral_acceptence_status = "accepted"
            session.commit()
            session.close()
            print('done')
            break

if __name__ == "__main__":
    
    referral_id_from_flask = sys.argv[1]
    comment = sys.argv[2]
    main(referral_id_from_flask, comment)

完整堆栈跟踪 #

[2023-04-17 09:35:13,375: ERROR/ForkPoolWorker-2] Traceback (most recent call last):
  File "/home/fusion/scrapers/care_portal/accept.py", line 43, in <module>
    driver = webdriver.Firefox(service=service, options=options)
  File "/home/fusion/flaskapp/venv/lib/python3.10/site-packages/selenium/webdriver/firefox/webdriver.py", line 199, in __init__
    super().__init__(command_executor=executor, options=options, keep_alive=True)
  File "/home/fusion/flaskapp/venv/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 286, in __init__
    self.start_session(capabilities, browser_profile)
  File "/home/fusion/flaskapp/venv/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 378, in start_session
    response = self.execute(Command.NEW_SESSION, parameters)
  File "/home/fusion/flaskapp/venv/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 440, in execute
    self.error_handler.check_response(response)
  File "/home/fusion/flaskapp/venv/lib/python3.10/site-packages/selenium/webdriver/remote/errorhandler.py", line 245, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 127

原因分析与解决方法 #

错误码127表示找不到执行依赖文件或权限不足，针对Celery+无头Firefox场景，按以下步骤排查修复：

1. 验证路径与执行权限 #

• 检查Firefox二进制文件是否存在且可执行：

  ls -l /usr/bin/firefox
  /usr/bin/firefox --version
  

• 确保Geckodriver有执行权限：

  chmod +x /home/fusion/bin/geckodriver
  /home/fusion/bin/geckodriver --version
  

• 确认Celery运行用户（如www-data、部署用户）拥有这两个文件的执行权限。

2. 安装无头Firefox依赖库 #

无头模式下Firefox需要额外系统依赖，Debian/Ubuntu系统执行：

apt-get install -y libgtk-3-0 libdbus-glib-1-2 libxt6 libx11-xcb1 libxcomposite1 libxdamage1 libxfixes3 libxi6 libxrandr2 libasound2 libpangocairo-1.0-0 libatk1.0-0 libcairo2 libcups2 libdrm2 libgbm1 libnss3 libxss1

3. 优化Firefox无头配置 #

替换旧版无头参数，添加适配服务器环境的配置：

options = Options()
options.add_argument("--headless=new")  # Firefox 102+推荐的新版无头模式
options.add_argument("--window-size=1920,1080")
options.add_argument("--disable-gpu")
options.add_argument("--no-sandbox")  # 规避沙箱权限限制
options.add_argument("--disable-dev-shm-usage")  # 解决临时空间不足问题
options.binary_location = '/usr/bin/firefox'

4. 调整WebDriver初始化位置 #

当前代码在模块全局初始化driver，Celery多进程环境会导致资源冲突，应将driver初始化移到main函数内部，并确保任务结束后关闭：

def main(referral_id_from_flask, comment):
    # 初始化driver
    options = Options()
    options.add_argument("--headless=new")
    options.add_argument("--window-size=1920,1080")
    options.add_argument("--disable-gpu")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    options.binary_location = '/usr/bin/firefox'
    service = Service(executable_path='/home/fusion/bin/geckodriver')
    driver = webdriver.Firefox(service=service, options=options)
    
    try:
        # 原有业务逻辑
        driver.get("https://www.example.com")
        # ...后续操作
    finally:
        driver.quit()  # 确保任务结束后关闭浏览器

5. 确认Celery环境变量配置 #

确保Celery进程能正确读取.env文件，或在Celery启动时显式传递环境变量，避免路径、账号等配置读取失败。

内容的提问来源于stack exchange，提问作者Mike C.