Ubuntu Server下Python Selenium无头无检测Chrome代理认证自动化方案

• 使用Python Selenium自动化访问网站时，触发「此浏览器或应用可能不安全，请更换浏览器」限制，改用undetected-chromedriver搭配代理解决
• 代理需身份认证，非无头模式下通过PyAutoGUI填充账号密码可完成认证，流程正常
• 在Ubuntu Server无头模式运行时，PyAutoGUI失效，无法处理代理认证弹窗，导致页面加载超时触发TimeoutException
• 尝试过undetected-chromedriver扩展、selenium wire代理配置等方案，均无法同时满足「无检测浏览器、带代理、无头运行」三个要求，且Selenium无法识别代理认证弹窗

当前伪代码 #

import pandas as pd
from imap_tools import MailBox, AND
import re
# No Headless, Proxy, undetected
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from threading import Thread
import pyautogui
from selenium.webdriver.chrome.options import Options
import chromedriver_autoinstaller
import undetected_chromedriver as uc
from seleniumwire import webdriver
from seleniumwire import webdriver

# host_ip = "xx.xxx.xx.xx"
host_ip = "x.xxx.xxx.xxx"
port = "xxxxx"
proxy_username = "xxxxxxxx"
proxy_password = "xxxxxxxx"
ipCheckURL="http://whatismyip.com"

URL=undetectedCheckUrl


def enter_proxy_auth(proxy_username, proxy_password):
    time.sleep(3)
    pyautogui.typewrite(proxy_username)
    pyautogui.press('tab')
    time.sleep(5)
    pyautogui.typewrite(proxy_password)
    pyautogui.press('enter')
    time.sleep(15)

def visitWebsiteAndAutomateAndReturnAValue(driver, url):
    driver.get(URL)
    #Do some Automation
    return Value

# 初始化驱动部分
chrome_options = Options()
chrome_options.add_argument('--proxy-server={}'.format(host_ip + ":" + port))
ucdriver = uc.Chrome(options=chrome_options, use_subprocess=True)
x1 = Thread(target=visitWebsiteAndAutomateAndReturnAValue, args=(ucdriver, url))
x2 = Thread(target=enter_proxy_auth, args=(proxy_username, proxy_password))
x1.start()
x2.start()
value = x1.join()
x2.join()
print(value)

方案1：将代理认证信息嵌入代理URL（最简便） #

直接把用户名和密码拼接进代理服务器地址，无需处理弹窗，适用于大多数支持URL认证的代理服务。

代码示例 #

import undetected_chromedriver as uc
from selenium.webdriver.chrome.options import Options

host_ip = "x.xxx.xxx.xxx"
port = "xxxxx"
proxy_username = "xxxxxxxx"
proxy_password = "xxxxxxxx"
URL = "目标网站地址"

def visitWebsiteAndAutomateAndReturnAValue(driver, url):
    driver.get(url)
    # 执行自动化操作
    return "处理结果"

# 配置无头模式和代理
chrome_options = Options()
# Ubuntu Server无头模式必备参数
chrome_options.add_argument('--headless=new')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
# 嵌入认证信息的代理地址
proxy_url = f"http://{proxy_username}:{proxy_password}@{host_ip}:{port}"
chrome_options.add_argument(f'--proxy-server={proxy_url}')

# 初始化undetected-chromedriver
ucdriver = uc.Chrome(options=chrome_options, use_subprocess=True)
try:
    result = visitWebsiteAndAutomateAndReturnAValue(ucdriver, URL)
    print(result)
finally:
    ucdriver.quit()

方案2：使用Chrome扩展自动处理代理认证 #

如果代理不支持URL嵌入认证，可以通过自定义Chrome扩展自动填充代理认证信息，配合undetected-chromedriver无头模式运行。

步骤1：创建扩展文件 #

创建名为proxy_auth_extension的文件夹，包含以下两个文件：

manifest.json

{
  "manifest_version": 3,
  "name": "Proxy Auth AutoFill",
  "version": "1.0",
  "permissions": ["webRequest", "webRequestAuthProvider", "<all_urls>"],
  "background": {
    "service_worker": "background.js"
  },
  "minimum_chrome_version": "109"
}

background.js

chrome.webRequest.onAuthRequired.addListener(
  function(details, callbackFn) {
    callbackFn({
      authCredentials: {
        username: "你的代理用户名",
        password: "你的代理密码"
      }
    });
  },
  {urls: ["<all_urls>"]},
  ["blocking"]
);

步骤2：在undetected-chromedriver中加载扩展 #

import undetected_chromedriver as uc
from selenium.webdriver.chrome.options import Options
import os

host_ip = "x.xxx.xxx.xxx"
port = "xxxxx"
URL = "目标网站地址"
# 扩展文件夹路径
extension_path = "./proxy_auth_extension"

def visitWebsiteAndAutomateAndReturnAValue(driver, url):
    driver.get(url)
    # 执行自动化操作
    return "处理结果"

# 配置无头模式和代理
chrome_options = Options()
chrome_options.add_argument('--headless=new')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument(f'--proxy-server={host_ip}:{port}')
# 加载自定义扩展
chrome_options.add_argument(f'--load-extension={os.path.abspath(extension_path)}')

# 初始化undetected-chromedriver
ucdriver = uc.Chrome(options=chrome_options, use_subprocess=True)
try:
    result = visitWebsiteAndAutomateAndReturnAValue(ucdriver, URL)
    print(result)
finally:
    ucdriver.quit()

方案3：结合undetected-chromedriver与Selenium Wire配置代理认证 #

通过Selenium Wire的代理配置直接传入认证信息，同时保持undetected-chromedriver的反检测特性。

代码示例 #

import undetected_chromedriver as uc
from selenium.webdriver.chrome.options import Options

host_ip = "x.xxx.xxx.xxx"
port = "xxxxx"
proxy_username = "xxxxxxxx"
proxy_password = "xxxxxxxx"
URL = "目标网站地址"

def visitWebsiteAndAutomateAndReturnAValue(driver, url):
    driver.get(url)
    # 执行自动化操作
    return "处理结果"

# 配置无头模式
chrome_options = Options()
chrome_options.add_argument('--headless=new')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')

# Selenium Wire代理配置
proxy_options = {
    'proxy': {
        'http': f'http://{host_ip}:{port}',
        'https': f'http://{host_ip}:{port}',
        'no_proxy': 'localhost,127.0.0.1',
        'username': proxy_username,
        'password': proxy_password
    }
}

# 初始化带Selenium Wire的undetected-chromedriver
ucdriver = uc.Chrome(options=chrome_options, seleniumwire_options=proxy_options, use_subprocess=True)
try:
    result = visitWebsiteAndAutomateAndReturnAValue(ucdriver, URL)
    print(result)
finally:
    ucdriver.quit()

内容的提问来源于stack exchange，提问作者faaiz hashmi