Selenium爬取Beatport Top100时元素定位失败问题求助

问题概述 #

开发Beatport Top100页面爬虫时，根元素div#__next可正常定位，但深层歌曲列表容器元素定位失败，抛出TimeoutException。

原代码 #

def scrape_beatport():
    user_agent = UserAgent().random
    chrome_options = Options()
    chrome_options.add_argument(f"user-agent={user_agent}")
    driver = webdriver.Chrome(options=chrome_options)
    
    try:
        driver.get('https://www.beatport.com/top-100')
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div#__next')))
        
        test1 = driver.find_elements(By.CSS_SELECTOR, 'div#__next')
        test2 = driver.fin_elements(By.CSS_SELECTOR, 'div.TracksList-style__Wrapper-sc-3fb03d50-8 bWXYsy row')
        
    finally:
        driver.quit()


if __name__ == '__main__':
    scrape_beatport()    

报错信息 #

Traceback (most recent call last):
  File "/Users/just/Documents/python/yt_test.py", line 84, in <module>
    scrape_beatport()    
    ^^^^^^^^^^^^^^^^^
  File "/Users/just/Documents/python/yt_test.py", line 71, in scrape_beatport
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'div.__next div.MainLayout-style__MainWrapper-sc-9f30c253-0 div div.MainLayout-style__Main-sc-9f30c253-1 main.MainLayout-style__MainContent-sc-9f30c253-2 div.TracksList-style__Wrapper-sc-3fb03d50-8')))
  File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/selenium/webdriver/support/wait.py", line 105, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: 
Stacktrace:
0   chromedriver                        0x0000000105272940 chromedriver + 4368704
1   chromedriver                        0x000000010526add4 chromedriver + 4337108
2   chromedriver                        0x0000000104e8ec04 chromedriver + 289796
3   chromedriver                        0x0000000104ed0e00 chromedriver + 560640
4   chromedriver                        0x0000000104f095ec chromedriver + 792044
5   chromedriver                        0x0000000104ec5ab4 chromedriver + 514740
6   chromedriver                        0x0000000104ec650c chromedriver + 517388
7   chromedriver                        0x0000000105236e5c chromedriver + 4124252
8   chromedriver                        0x000000010523bc4c chromedriver + 4144204
9   chromedriver                        0x000000010521c824 chromedriver + 4016164
10  chromedriver                        0x000000010523c57c chromedriver + 4146556
11  chromedriver                        0x000000010520e2d8 chromedriver + 3957464
12  chromedriver                        0x000000010525bec4 chromedriver + 4275908
13  chromedriver                        0x000000010525c040 chromedriver + 4276288
14  chromedriver                        0x000000010526aa34 chromedriver + 4336180
15  libsystem_pthread.dylib             0x000000018a0bd034 _pthread_start + 136
16  libsystem_pthread.dylib             0x000000018a0b7e3c thread_start + 8

问题分析与修复方案 #

1. 拼写错误 #

原代码中driver.fin_elements是笔误，正确应为driver.find_elements。

2. CSS选择器错误 #

原选择器div.TracksList-style__Wrapper-sc-3fb03d50-8 bWXYsy row格式错误：bWXYsy和row是元素的类名，需用.连接，正确写法为div.TracksList-style__Wrapper-sc-3fb03d50-8.bWXYsy.row。但Beatport使用动态生成的类名，会随页面更新变化，建议改用稳定的属性选择器，比如页面中歌曲列表容器的data-testid="tracks-list"属性，选择器为div[data-testid="tracks-list"]，避免类名变更导致定位失败。

3. 等待逻辑优化 #

原代码仅等待根元素存在，深层元素可能未完成渲染。应直接等待目标元素的可见性（而非仅存在），使用EC.visibility_of_element_located替代EC.presence_of_element_located，确保元素已渲染完成。

4. 反爬应对 #

Beatport会检测爬虫，可添加页面滚动、隐式等待，或处理可能出现的Cookie弹窗，提升爬取稳定性。

修正后的代码 #

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from fake_useragent import UserAgent

def scrape_beatport():
    user_agent = UserAgent().random
    chrome_options = Options()
    chrome_options.add_argument(f"user-agent={user_agent}")
    # 可选：禁用图片加载提升速度
    chrome_options.add_argument("--blink-settings=imagesEnabled=false")
    
    driver = webdriver.Chrome(options=chrome_options)
    driver.implicitly_wait(5)  # 全局隐式等待
    
    try:
        driver.get('https://www.beatport.com/top-100')
        
        # 处理Cookie弹窗（若存在）
        try:
            accept_btn = WebDriverWait(driver, 5).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, 'button[data-testid="cookie-accept"]'))
            )
            accept_btn.click()
        except:
            pass
        
        # 等待歌曲列表容器可见
        tracks_container = WebDriverWait(driver, 15).until(
            EC.visibility_of_element_located((By.CSS_SELECTOR, 'div[data-testid="tracks-list"]'))
        )
        
        # 验证定位结果
        test1 = driver.find_elements(By.CSS_SELECTOR, 'div#__next')
        test2 = driver.find_elements(By.CSS_SELECTOR, 'div[data-testid="tracks-list"]')
        
        print(f"test1找到元素数量：{len(test1)}")
        print(f"test2找到元素数量：{len(test2)}")
        
        # 后续可在此处提取歌曲信息
        
    finally:
        driver.quit()

if __name__ == '__main__':
    scrape_beatport()    

内容的提问来源于stack exchange，提问作者Just Verhagen