Selenium处理不完整URL及通用验证码检测方案咨询

一、解决URL协议（HTTP/HTTPS）自动适配问题 #

原代码仅添加了www前缀，但缺少协议部分导致Selenium无法正常访问。可以通过优先尝试HTTPS，失败后自动降级到HTTP的逻辑处理，具体实现如下：

import requests
from selenium import webdriver
from selenium.common.exceptions import WebDriverException

# 保留原数据库连接逻辑
db_conn = base_class.table_selected_urls()
db_conn.execute("SELECT DISTINCT Domain,Site_name FROM URLS_INFO")
urls = db_conn.fetchall()

def get_valid_url(url_name):
    # 处理www前缀
    if url_name.split('.')[0] != "www":
        url_name = f"www.{url_name}"
    # 先尝试HTTPS协议
    https_url = f"https://{url_name}"
    try:
        # 用requests快速检测连接状态，超时设为5秒
        response = requests.get(https_url, timeout=5)
        # 只要能建立连接（非400+的致命错误）就算有效
        if response.status_code in [200, 403, 404]:
            return https_url
    except (requests.exceptions.RequestException, ConnectionError):
        # HTTPS失败，尝试HTTP协议
        http_url = f"http://{url_name}"
        try:
            response = requests.get(http_url, timeout=5)
            if response.status_code in [200, 403, 404]:
                return http_url
        except:
            # 两种协议都无法访问，标记为无效URL
            return None

for url in urls:
    url_name = url[0]
    url_source = url[1]
    valid_url = get_valid_url(url_name)
    if valid_url:
        print(f"可用URL: {valid_url}")
        # 后续在这里调用Selenium访问valid_url即可
    else:
        print(f"无法访问: {url_name}")

说明：用requests快速检测比直接启动Selenium更高效，能节省资源；如果必须用Selenium访问，也可以捕获WebDriverException后切换协议重试。

二、通用验证码检测方案 #

针对不同网站验证码DOM标识不统一的问题，可从关键词匹配、元素特征、视觉识别三个维度组合判断：

1. 基于DOM元素的关键词检测 #

遍历页面核心元素，检查其属性是否包含验证码相关关键词（中英文结合）：

from selenium.webdriver.common.by import By

def has_captcha(driver):
    # 验证码相关关键词列表
    captcha_keywords = ["captcha", "verify", "validation", "code", "验证码", "验证", "校验"]
    
    # 检查图片验证码（img标签）
    img_elements = driver.find_elements(By.TAG_NAME, "img")
    for img in img_elements:
        alt_text = img.get_attribute("alt") or ""
        img_src = img.get_attribute("src") or ""
        if any(keyword in alt_text.lower() or keyword in img_src.lower() for keyword in captcha_keywords):
            return True
    
    # 检查验证码输入框（input标签）
    input_elements = driver.find_elements(By.TAG_NAME, "input")
    for inp in input_elements:
        placeholder = inp.get_attribute("placeholder") or ""
        class_attr = inp.get_attribute("class") or ""
        id_attr = inp.get_attribute("id") or ""
        if any(keyword in placeholder.lower() or keyword in class_attr.lower() or keyword in id_attr.lower() for keyword in captcha_keywords):
            return True
    
    # 检查页面文本中的验证码提示
    page_text = driver.page_source.lower()
    if any(keyword in page_text for keyword in captcha_keywords):
        return True
    
    # 检查行为验证码特征（如滑块、canvas元素）
    canvas_elements = driver.find_elements(By.TAG_NAME, "canvas")
    for canvas in canvas_elements:
        class_attr = canvas.get_attribute("class") or ""
        if any(keyword in class_attr.lower() for keyword in captcha_keywords):
            return True
    
    return False

2. 可选：结合OCR的视觉检测 #

如果DOM检测失效，可通过OCR识别图片内容判断是否为验证码：

import pytesseract
from PIL import Image
from io import BytesIO
import requests

def has_captcha_ocr(driver):
    captcha_keywords = ["captcha", "验证码", "verify"]
    img_elements = driver.find_elements(By.TAG_NAME, "img")
    for img in img_elements:
        img_src = img.get_attribute("src")
        if img_src:
            try:
                response = requests.get(img_src)
                img_data = Image.open(BytesIO(response.content))
                # 识别图片文本
                ocr_text = pytesseract.image_to_string(img_data).lower()
                if any(keyword in ocr_text for keyword in captcha_keywords):
                    return True
            except:
                continue
    return False

3. 检测结果存储 #

存入CSV示例

import csv

# 初始化CSV文件
with open("captcha_detection_result.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.writer(f)
    writer.writerow(["URL", "站点名称", "是否含验证码"])

    for url in urls:
        url_name = url[0]
        url_source = url[1]
        valid_url = get_valid_url(url_name)
        if valid_url:
            driver = webdriver.Chrome()
            driver.get(valid_url)
            has_capt = has_captcha(driver)
            writer.writerow([valid_url, url_source, "是" if has_capt else "否"])
            driver.quit()
        else:
            writer.writerow([url_name, url_source, "无法访问"])

存入数据库示例

# 假设数据库存在结果表CAPTCHA_RESULTS，字段为URL, Site_name, Has_Captcha
for url in urls:
    url_name = url[0]
    url_source = url[1]
    valid_url = get_valid_url(url_name)
    if valid_url:
        driver = webdriver.Chrome()
        driver.get(valid_url)
        has_capt = has_captcha(driver)
        # 插入数据库
        db_conn.execute("INSERT INTO CAPTCHA_RESULTS (URL, Site_name, Has_Captcha) VALUES (%s, %s, %s)", 
                        (valid_url, url_source, has_capt))
        db_conn.commit()
        driver.quit()
    else:
        db_conn.execute("INSERT INTO CAPTCHA_RESULTS (URL, Site_name, Has_Captcha) VALUES (%s, %s, %s)", 
                        (url_name, url_source, "无法访问"))
        db_conn.commit()

内容的提问来源于stack exchange，提问作者stack overflow