如何用Selenium抓取含可折叠字段的页面？以FlashScore为例

我需要抓取https://www.flashscore.com/的赛事数据，包括赛事名称、球队名称和日期。目前已开发基础抓取方案，但遇到可折叠赛事板块的问题——比如安道尔杯板块有触发按钮，必须点击激活才能抓取到该板块下的赛事数据，因此需要自动化点击所有此类按钮以展开字段完成抓取。

我先后尝试两种方案均失败：

• 查找class为wclIcon__leagueShowMoreCont的按钮，修改其内部元素的data-state为open，无效。
• 通过XPath定位包含display matches文本的元素并逐个点击，仍未解决问题。

初始尝试代码 #

url = 'https://www.flashscore.com/'  # Replace this with the URL of the website

# Open the website
driver.get(url)

# Wait for the calendar date picker button to be clickable
button = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.CLASS_NAME, 'calendar__datepicker')))
button_text = button.text.strip()

all_data = []
for _ in range(1):  # Click the "Next day" button three times
    
    next_day_button = WebDriverWait(driver, 50).until(EC.element_to_be_clickable((By.CLASS_NAME, 'calendar__navigation--tomorrow')))
    
    # Dismiss the cookie banner if it appears
    try:
        cookie_banner = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.ID, 'onetrust-banner-sdk')))
        cookie_banner.click()  # Click to dismiss the cookie banner if found
    except:
        pass  # If the cookie banner doesn't appear or can't be dismissed
    
    driver.execute_script("arguments[0].click();", next_day_button)
    
    # Wait for the loading overlay to disappear after clicking
    wait = WebDriverWait(driver, 30)
    wait.until(EC.invisibility_of_element_located((By.CLASS_NAME, 'loadingOverlay')))
    
    next_day_button = WebDriverWait(driver, 50).until(EC.element_to_be_clickable((By.CLASS_NAME, 'calendar__navigation--tomorrow')))
    
    # Dismiss the cookie banner if it appears
    try:
        cookie_banner = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.ID, 'onetrust-banner-sdk')))
        cookie_banner.click()  # Click to dismiss the cookie banner if found
    except:
        pass  # If the cookie banner doesn't appear or can't be dismissed
    
    driver.execute_script("arguments[0].click();", next_day_button)
    
    # Wait for the loading overlay to disappear after clicking
    wait = WebDriverWait(driver, 30)
    wait.until(EC.invisibility_of_element_located((By.CLASS_NAME, 'loadingOverlay')))

    # Get the date for each day
    date_element = driver.find_element(By.CLASS_NAME, 'calendar__datepicker')
    date = date_element.text.strip()
    print("Date:", date)

    # Get the page source after loading the subsequent days
    html_content = driver.page_source

    # Use BeautifulSoup to parse the HTML content
    soup = BeautifulSoup(html_content, 'html.parser')

    # Find all event containers
    event_containers = soup.find_all('div', class_='event__match')
    
    data = []

    for event in event_containers:
        # Extract event type
        event_type = event.find_previous('div', class_='event__title')
        
        button_check = event.find_previous('div', class_='wclIcon__leagueShowMoreCont')
        
        trigger_element = button_check.find(class_='_trigger_9ionf_42')
        
        if trigger_element:
            trigger_element['data-state'] = 'open'
            
        print(button_check.prettify())
        
        event_type_text = event_type.find(class_='event__titleInfo').get_text(strip=True)
        
        # Extract event name
        event_name = event_type.find(class_='event__titleBox').get_text(strip=True)
        
        # Extract event time
        event_time_elem = event.find(class_='event__time')
        event_time = event_time_elem.get_text(strip=True) if event_time_elem else 'No time available'
        
        # Extract teams
        away_team = event.find(class_='event__participant--away').get_text(strip=True)
        home_team = event.find(class_='event__participant--home').get_text(strip=True)
        
        data.append({
            'Date': date,
            'Country': event_type_text,
            'Event Name': event_name,
            'Event Time': event_time,
            'Home Team': home_team,
            'Away Team': away_team
        })
        
        all_data.extend(data)

# Create a DataFrame using pandas
df = pd.DataFrame(all_data)

更新后的尝试代码 #

WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.ID, 'onetrust-banner-sdk')))

wait = WebDriverWait(driver, 10)

# elements = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'display matches')]")))
elements = WebDriverWait(driver, 10).until(EC.visibility_of_all_elements_located((By.XPATH, "//span[contains(text(),'display matches')]")))

# Click on each of the collapsible element and expand it
for element in elements:
    element.click()

内容的提问来源于stack exchange，提问作者Shruthi Ravishankar