Python Selenium爬取CAISO太阳能CSV数据遇点击失效求助

1. 改用显式等待+可靠定位方式 #

绝对XPath极度脆弱，页面结构稍有变动就会失效，建议用元素唯一的id（downloadRenewablesCSV）定位，同时配合显式等待确保元素加载完成并可点击：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()  # Selenium 4+版本无需手动指定executable_path，会自动匹配驱动
driver.get('https://www.caiso.com/TodaysOutlook/Pages/supply.html#section-renewables-trend')

# 最长等待10秒，直到元素可点击
csv_button = WebDriverWait(driver, 10).until(
    EC.element_to_be_clickable((By.ID, 'downloadRenewablesCSV'))
)
csv_button.click()

2. 先展开下拉菜单（必要时） #

观察页面可知，"Chart data (CSV)"选项嵌套在下拉菜单中，可能需要先点击下拉触发按钮展开菜单，才能点击CSV选项：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get('https://www.caiso.com/TodaysOutlook/Pages/supply.html#section-renewables-trend')

# 先等待并点击下拉菜单触发按钮
export_dropdown = WebDriverWait(driver, 10).until(
    EC.element_to_be_clickable((By.XPATH, '//button[contains(@class, "dropdown-toggle") and contains(text(), "Export")]'))
)
export_dropdown.click()

# 再等待并点击CSV下载按钮
csv_button = WebDriverWait(driver, 10).until(
    EC.element_to_be_clickable((By.ID, 'downloadRenewablesCSV'))
)
csv_button.click()

3. 用JavaScript强制触发点击 #

如果常规点击无效，大概率是元素的点击事件绑定在JavaScript逻辑上，直接执行JS点击更可靠：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get('https://www.caiso.com/TodaysOutlook/Pages/supply.html#section-renewables-trend')

csv_button = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'downloadRenewablesCSV'))
)
# 执行JS代码触发点击
driver.execute_script("arguments[0].click();", csv_button)

4. 直接提取CSV链接下载（跳过点击交互） #

点击后元素会动态生成href属性，可等待属性生成后直接提取链接，用requests下载数据，避免处理浏览器下载弹窗：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests

driver = webdriver.Chrome()
driver.get('https://www.caiso.com/TodaysOutlook/Pages/supply.html#section-renewables-trend')

csv_button = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'downloadRenewablesCSV'))
)
# 触发href生成
driver.execute_script("arguments[0].click();", csv_button)

# 等待href属性生成
WebDriverWait(driver, 10).until(
    lambda d: csv_button.get_attribute('href') is not None
)
csv_url = csv_button.get_attribute('href')

# 下载CSV数据到本地
response = requests.get(csv_url)
with open('renewables_solar_data.csv', 'wb') as f:
    f.write(response.content)

driver.quit()

内容的提问来源于stack exchange，提问作者Zachary McArthur