Python Selenium操控Chrome时页面无法加载，疑似JS执行异常

我想用Python Selenium自动化完成https://haveibeensquatted.com/上的域名检测流程，但遇到了问题——用Selenium运行时页面无法加载检测结果，手动操作等待却一切正常。以下是我的代码：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import csv
import time

# === CONFIGURATION ===
INPUT_DOMAINS = [
    "example.com",
    "nonexistent-domain-xyz123.org",
    # add more domains here
]
OUTPUT_CSV = "squatted_results.csv"
URL = "https://haveibeensquatted.com/"

# === SET UP SELENIUM WITH CHROME ===
options = webdriver.ChromeOptions()
service = Service(ChromeDriverManager().install())

driver = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(driver, 20)
results = []

# XPaths
RESULTS_READY_XPATH = "/html/body/main/div[3]/header/div/div/div[2]/div[2]/span[2]"
ITEMS_XPATH = "/html/body/main/div[1]/div[3]/div/div/div/form/following-sibling::div//ul/li"
NO_SQUAT_XPATH = "//div[contains(text(),'No squatted permutations')]"
TABLE_CELL_XPATH = "/html/body/main/div[3]/div/div/div[2]/div/table/tbody/tr/td"

try:
    for domain in INPUT_DOMAINS:
        # Load fresh page for each domain
        driver.get(URL)

        # Enter domain
        input_xpath = "/html/body/main/div[1]/div[3]/div/div/div/form/div[2]/input"
        btn_xpath = "/html/body/main/div[1]/div[3]/div/div/div/form/button"
        input_box = wait.until(EC.element_to_be_clickable((By.XPATH, input_xpath)))
        input_box.clear()
        input_box.send_keys(domain)
        driver.find_element(By.XPATH, btn_xpath).click()
        time.sleep(1000)

finally:
    driver.quit()

# Save results to CSV
with open(OUTPUT_CSV, "w", newline='', encoding="utf-8") as f:
    writer = csv.writer(f)
    writer.writerow(["domain", "status"])
    writer.writerows(results)

print(f"Done! Results saved to '{OUTPUT_CSV}'")

问题原因 #

1. 不合理的等待方式：time.sleep(1000)是固定超长等待，不仅完全没必要，还可能因为页面异步请求未完成就继续执行（或超时），导致结果无法正确加载；更关键的是当前代码根本没有提取结果的逻辑，自然无法生成有效CSV。
2. 自动化识别拦截：网站可能通过检测Selenium的默认特征（如navigator.webdriver标识）限制自动化请求，导致检测流程无法正常触发。
3. 未使用正确的等待条件：页面检测结果是动态生成的，需要等待特定元素出现来确认结果加载完成，而非盲目等待。

修复后的完整代码 #

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import csv

# === CONFIGURATION ===
INPUT_DOMAINS = [
    "example.com",
    "nonexistent-domain-xyz123.org",
    # 添加更多域名
]
OUTPUT_CSV = "squatted_results.csv"
URL = "https://haveibeensquatted.com/"

# === SET UP SELENIUM WITH CHROME ===
options = webdriver.ChromeOptions()
# 禁用自动化标识，避免被网站检测
options.add_argument("--disable-blink-features=AutomationControlled")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
# 模拟正常浏览器窗口
options.add_argument("--start-maximized")

service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(driver, 30)
results = []

try:
    for domain in INPUT_DOMAINS:
        driver.get(URL)
        
        # 输入域名并提交
        input_box = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='domain']")))
        input_box.clear()
        input_box.send_keys(domain)
        submit_btn = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']")))
        submit_btn.click()
        
        try:
            # 等待无 squat 结果的提示
            wait.until(EC.visibility_of_element_located((By.XPATH, "//div[contains(text(),'No squatted permutations')]")))
            results.append([domain, "无 squat 域名"])
        except TimeoutException:
            # 如果无结果提示没出现，等待结果列表加载
            wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "ul.list-group")))
            squat_items = driver.find_elements(By.CSS_SELECTOR, "ul.list-group li")
            # 收集所有squat域名，用分号分隔
            squat_domains = "; ".join([item.text.strip() for item in squat_items])
            results.append([domain, f"检测到 squat 域名: {squat_domains}"])

finally:
    driver.quit()

# 保存结果到CSV
with open(OUTPUT_CSV, "w", newline='', encoding="utf-8") as f:
    writer = csv.writer(f)
    writer.writerow(["域名", "检测状态"])
    writer.writerows(results)

print(f"完成！结果已保存到 '{OUTPUT_CSV}'")

关键修改说明 #

• 反检测配置：添加了禁用自动化标识的选项，让Selenium模拟正常浏览器行为，避免被网站拦截。
• 替换XPATH为CSS选择器：CSS选择器比硬编码的绝对XPATH更稳定，不易因页面结构微小变动失效。
• 智能等待逻辑：先尝试等待无结果提示，超时则认为存在squat结果，再等待结果列表加载，确保结果正确提取。
• 结果提取与存储：完善了结果收集逻辑，将检测到的squat域名或无结果状态存入列表，最终生成有效CSV。

内容的提问来源于stack exchange，提问作者mazix