Selenium解析Avito网站失败报错，ChromeDriver异常关闭求助

• 原本使用BeautifulSoup解析Avito网站，改用Selenium重构后出现两个核心问题：
  1. ChromeDriver启动后立即关闭，同时抛出错误
  2. 解析初期控制台输出大量重复字符，最终仅打印目标链接，无正常解析结果

相关代码：

# aiogram
from aiogram import types
from aiogram.types.message import ParseMode
from bot import dp
from bot import db
from aiogram.utils.markdown import hbold, hlink
import cfscrape

import fake_headers

# python
import re
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium_stealth import stealth
from selenium.webdriver.common.by import By
import time

# default для mvc
city = "belgorod"
city2 = "voronezh"
city3 = "kursk"

radius = 300
allowed_data = ['часов', 'часа','час']

options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options)

stealth(
        driver,
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
)

  @dp.message_handler(lambda message: message.text == "Воронеж")
    async def avito_list(message: types.Message):
        try:
            # Проверим есть ли вообще подписки
            if db.follows_exists(message.from_user.id):

                # Выводим все объявления по подпискам
                follows = db.show_subs(message.from_user.id)

                for follow in follows:
                    line = follow[1][0][0]

                    with db.connection:
                        marka = db.cursor.execute(f"SELECT `avito_mark_name` FROM `marks` WHERE `name` = ?", (line,)).fetchall()
                        marka = marka[0][0]

                    min_price = follow[2][0][0]
                    if follow[3][0][0] == None:
                        max_price = follow[2][0][0]*1000
                    else:
                        max_price = follow[3][0][0]
                    #model = "2114_samara"
                    #url = f"https://www.avito.ru/{city}/avtomobili/{marka}/{model}?radius={radius}"
                    url = f"https://www.avito.ru/valuyki/avtomobili/{marka}-ASgBAgICAUTgtg3GmSg?cd=1&radius=200"
                    driver.get(url)
                    time.sleep(5)
                    print(url)

                    main_container = driver.find_elements(By.CSS_SELECTOR,".iva-item-content")

                    for index, content in enumerate(main_container):
                        contaier_of_content = content.find_element(By.CSS_SELECTOR, ".iva-item-body")



                        ad_post = contaier_of_content.find_element(By.CLASS_NAME, "item-line")


                        #fresh_car = contaier_of_content.find_element(By.CLASS_NAME,{"data-marker":"item-date"}).text.split()[1]
                        price = content.find_element(By.CSS_SELECTOR, ".price-price").find_element(By.CLASS_NAME, "price").get_attribute("content")

                        if int(price) <= max_price and int(price) >= min_price: # проверка на сегодняшний день и на подхождение по цене



                            title_info = contaier_of_content.find_element(By.CLASS_NAME,'.iva-item-title').get_attribute("title").split(',')
                            town_info = contaier_of_content.find_element(By.CLASS_NAME,'.geo-root').find_element(By.TAG_NAME,"span").text
                            datePost_info = contaier_of_content.find_element(By.CLASS_NAME,'.iva-item-dateInfo').find_element(By.TAG_NAME,"div").text
                            #link = content.find('a', class_=re.compile('iva-item-sliderLink*'))['href']

                            currency = content.find_element(By.CSS_SELECTOR,'.price-price').find_element(By.TAG_NAME,'meta').get_attribute("content")
                            car_info = title_info[0]
                            was_created = title_info[1]
                            city_on_sale = town_info

                            card = f'{hlink(car_info+" - "+was_created,"https://avito.ru"+link)}\n' \
                                f'{hbold("Город: ", city_on_sale)}\n' \
                                f'{hbold("Цена: ", price, currency)}'



                            await message.answer(card)
        except Exception as ex:
            print(ex)
        finally:
            driver.close()
            driver.quit()

控制台仅输出：

https://www.avito.ru/valuyki/avtomobili/volkswagen-ASgBAgICAUTgtg3GmSg?cd=1&radius=200
https://www.avito.ru/valuyki/avtomobili/vaz_lada-ASgBAgICAUTgtg3GmSg?cd=1&radius=200

1. ChromeDriver提前关闭的原因 #

driver.close()和driver.quit()放在了finally块中，不管解析流程是否完成都会执行，导致浏览器刚启动就被销毁。

修复方式：

• 不在全局初始化driver，而是在命令处理函数内部创建实例
• 确保仅在整个解析流程结束后再关闭浏览器

2. 解析失败的核心问题 #

（1）选择器错误 #

代码中多处使用错误的元素定位规则：

• find_element(By.CLASS_NAME,'.iva-item-title')：CLASS_NAME不能带.前缀，需改为By.CSS_SELECTOR或去掉.
• link变量被注释，导致拼接链接时出现未定义错误
• 依赖类名定位不稳定，建议改用Avito页面的data-marker属性（如[data-marker="item-title"]），这类属性更不容易随页面更新变化

（2）页面加载等待不严谨 #

time.sleep(5)属于硬编码等待，无法适配网络波动或页面加载延迟，容易导致元素未加载完成就开始解析。

修复方式：
使用WebDriverWait等待目标元素加载完成，替代固定时长等待。

（3）异常处理过于宽泛 #

全局except Exception as ex会隐藏具体错误信息，无法定位问题根源。

修复方式：
捕获异常时打印完整堆栈信息，方便排查具体错误。

# aiogram
from aiogram import types
from aiogram.types.message import ParseMode
from bot import dp
from bot import db
from aiogram.utils.markdown import hbold, hlink
import cfscrape

import fake_headers

# python
import re
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium_stealth import stealth
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import traceback

# default для mvc
city = "belgorod"
city2 = "voronezh"
city3 = "kursk"

radius = 300
allowed_data = ['часов', 'часа','час']

@dp.message_handler(lambda message: message.text == "Воронеж")
async def avito_list(message: types.Message):
    driver = None
    try:
        # 初始化浏览器实例
        options = webdriver.ChromeOptions()
        options.add_argument("start-maximized")
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option('useAutomationExtension', False)
        driver = webdriver.Chrome(options=options)
        
        stealth(
            driver,
            languages=["en-US", "en"],
            vendor="Google Inc.",
            platform="Win32",
            webgl_vendor="Intel Inc.",
            renderer="Intel Iris OpenGL Engine",
            fix_hairline=True,
        )

        # 检查用户是否有订阅
        if db.follows_exists(message.from_user.id):
            follows = db.show_subs(message.from_user.id)

            for follow in follows:
                line = follow[1][0][0]

                with db.connection:
                    marka = db.cursor.execute(f"SELECT `avito_mark_name` FROM `marks` WHERE `name` = ?", (line,)).fetchall()
                    marka = marka[0][0]

                min_price = follow[2][0][0]
                max_price = follow[3][0][0] if follow[3][0][0] is not None else min_price * 1000
                
                url = f"https://www.avito.ru/valuyki/avtomobili/{marka}-ASgBAgICAUTgtg3GmSg?cd=1&radius=200"
                driver.get(url)
                print(url)

                # 等待主容器加载完成
                WebDriverWait(driver, 10).until(
                    EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".iva-item-content"))
                )

                main_container = driver.find_elements(By.CSS_SELECTOR,".iva-item-content")

                for content in main_container:
                    # 使用data-marker定位价格元素
                    price_elem = content.find_element(By.CSS_SELECTOR, '[data-marker="item-price"]')
                    price = price_elem.get_attribute("content")
                    
                    if not price:
                        continue
                    
                    price_int = int(price)
                    if min_price <= price_int <= max_price:
                        # 获取标题和链接
                        title_elem = content.find_element(By.CSS_SELECTOR, '[data-marker="item-title"]')
                        title = title_elem.get_attribute("title")
                        link = title_elem.get_attribute("href")

                        # 获取城市信息
                        town_elem = content.find_element(By.CSS_SELECTOR, '[data-marker="item-region"]')
                        town_info = town_elem.text

                        # 获取发布日期
                        date_elem = content.find_element(By.CSS_SELECTOR, '[data-marker="item-date"]')
                        datePost_info = date_elem.text

                        # 获取货币类型
                        currency = price_elem.find_element(By.TAG_NAME, 'meta').get_attribute("content")

                        # 拆分标题信息
                        title_parts = title.split(',', 1)
                        car_info = title_parts[0] if len(title_parts) > 0 else title
                        was_created = title_parts[1].strip() if len(title_parts) > 1 else ""

                        # 构建消息卡片
                        card = f'{hlink(f"{car_info} - {was_created}", link)}\n' \
                               f'{hbold("Город:")} {town_info}\n' \
                               f'{hbold("Цена:")} {price} {currency}'

                        await message.answer(card, parse_mode=ParseMode.HTML)
                        # 避免发送过快触发限制
                        time.sleep(1)
    except Exception as ex:
        # 打印完整错误堆栈
        traceback.print_exc()
        await message.answer(f"Ошибка при парсинге: {str(ex)}")
    finally:
        # 确保浏览器实例被正确关闭
        if driver:
            driver.quit()

• 优先使用WebDriverWait代替固定时长等待，提升解析稳定性
• 针对Avito的反爬机制，可增加随机延迟、轮换User-Agent
• 定期检查页面元素定位器，网站结构更新后及时调整
• 长期运行的机器人可启用无头浏览器模式（options.add_argument("--headless=new")），减少资源占用

内容的提问来源于stack exchange，提问作者Игорь Кудряшов