如何提取网页带onclick属性的tr元素数据并完善Selenium爬虫？

1. 修复核心语法错误 #

你的代码第17行及后续所有CSS选择器都存在未闭合方括号的语法问题，这是导致元素查找失败的直接原因：

• 错误写法：div[id*='ContentPlaceHolder1_divPageContent'
• 正确写法：div[id*='ContentPlaceHolder1_divPageContent']
  所有div[class*='xxx']的选择器都需要补上闭合的]。

2. 实现CSV追加写入 #

使用pandas的to_csv方法时，通过mode='a'开启追加模式，同时判断文件是否存在来决定是否写入表头，避免重复表头。

3. 添加爬取时间列 #

用time.strftime("%H:%M:%S")获取当前时间的hh:mm:ss格式，为爬取到的所有数据统一添加该时间列。

修改后的完整代码 #

import time 
import pandas as pd 
import os
from selenium import webdriver 
from selenium.webdriver import Chrome 
from selenium.webdriver.common.by import By

options = webdriver.ChromeOptions() 
options.add_argument("--headless")
options.page_load_strategy = "none"
driver = Chrome(options=options) 
driver.implicitly_wait(5)
url = "C:/Users/Mirko/Desktop/test.htm" 
driver.get(url) 
time.sleep(5)

# 修复CSS选择器的闭合括号
content = driver.find_element(By.CSS_SELECTOR, "div[id*='ContentPlaceHolder1_divPageContent']")
flights = content.find_elements(By.XPATH, ".//tr[@onclick]")

def extract_data(element): 
    # 所有CSS选择器都补上闭合的]，并添加空值判断避免报错
    antStatus = element.find_elements(By.CSS_SELECTOR, "div[class*='antStatus']")
    antStatus = antStatus[0].get_attribute("textContent") if antStatus else ""
    antFlight = element.find_elements(By.CSS_SELECTOR, "div[class*='antFlight']")
    antFlight = antFlight[0].get_attribute("textContent") if antFlight else ""
    antCS = element.find_elements(By.CSS_SELECTOR, "div[class*='antCS']")
    antCS = antCS[0].get_attribute("textContent") if antCS else ""
    antCity = element.find_elements(By.CSS_SELECTOR, "div[class*='antCity']")
    antCity = antCity[0].get_attribute("textContent") if antCity else ""
    antID = element.find_elements(By.CSS_SELECTOR, "div[class*='antID']")
    antID = antID[0].get_attribute("textContent") if antID else ""
    antSTD = element.find_elements(By.CSS_SELECTOR, "div[class*='antSTD']")
    antSTD = antSTD[0].get_attribute("textContent") if antSTD else ""
    antETD = element.find_elements(By.CSS_SELECTOR, "div[class*='antETD']")
    antETD = antETD[0].get_attribute("textContent") if antETD else ""
    antATD = element.find_elements(By.CSS_SELECTOR, "div[class*='antATD']")
    antATD = antATD[0].get_attribute("textContent") if antATD else ""
    antATT = element.find_elements(By.CSS_SELECTOR, "div[class*='antATT']")
    antATT = antATT[0].get_attribute("textContent") if antATT else ""
    antBAY = element.find_elements(By.CSS_SELECTOR, "div[class*='antBAY']")
    antBAY = antBAY[0].get_attribute("textContent") if antBAY else ""
    antGATE = element.find_elements(By.CSS_SELECTOR, "div[class*='antGATE']")
    antGATE = antGATE[0].get_attribute("textContent") if antGATE else ""
    antTYPE = element.find_elements(By.CSS_SELECTOR, "div[class*='antTYPE']")
    antTYPE = antTYPE[0].get_attribute("textContent") if antTYPE else ""
    antREG = element.find_elements(By.CSS_SELECTOR, "div[class*='antREG']")
    antREG = antREG[0].get_attribute("textContent") if antREG else ""
    antSU1 = element.find_elements(By.CSS_SELECTOR, "div[class*='antSU1']")
    antSU1 = antSU1[0].get_attribute("textContent") if antSU1 else ""
    antSU2 = element.find_elements(By.CSS_SELECTOR, "div[class*='antSU2']")
    antSU2 = antSU2[0].get_attribute("textContent") if antSU2 else ""
    antRMK = element.find_elements(By.CSS_SELECTOR, "div[class*='antRMK']")
    antRMK = antRMK[0].get_attribute("textContent") if antRMK else ""
    antPOZ = element.find_elements(By.CSS_SELECTOR, "div[class*='antPOZ']")
    antPOZ = antPOZ[0].get_attribute("textContent") if antPOZ else ""
    antBUS = element.find_elements(By.CSS_SELECTOR, "div[class*='antBUS']")
    antBUS = antBUS[0].get_attribute("textContent") if antBUS else ""
    antCheckin = element.find_elements(By.CSS_SELECTOR, "div[class*='antCheckin']")
    antCheckin = antCheckin[0].get_attribute("textContent") if antCheckin else ""
    antDeicing = element.find_elements(By.CSS_SELECTOR, "div[class*='antDeicing']")
    antDeicing = antDeicing[0].get_attribute("textContent") if antDeicing else ""

    return { 
        "antStatus": antStatus, 
        "antFlight": antFlight, 
        "antCS": antCS, 
        "antCity": antCity, 
        "antID": antID,
        "antSTD": antSTD,
        "antETD": antETD,
        "antATD": antATD,
        "antATT": antATT,
        "antBAY": antBAY,
        "antGATE": antGATE,
        "antTYPE": antTYPE,
        "antREG": antREG,
        "antSU1": antSU1,
        "antSU2": antSU2,
        "antRMK": antRMK,
        "antPOZ": antPOZ,
        "antBUS": antBUS,
        "antCheckin": antCheckin,
        "antDeicing": antDeicing
    }

data = [] 
 
for flight in flights: 
    extracted_data = extract_data(flight) 
    data.append(extracted_data) 

if data:
    df = pd.DataFrame(data)
    # 添加爬取时间列，格式为hh:mm:ss
    crawl_time = time.strftime("%H:%M:%S")
    df["爬取时间"] = crawl_time
    # 追加写入CSV，判断文件是否存在来决定是否写表头
    df.to_csv("ob.csv", mode='a', index=False, header=not os.path.exists("ob.csv"))

driver.quit()

额外优化说明 #

• 给每个元素查找添加了if xxx else ""的判断，避免因元素不存在导致索引越界报错
• 最后添加了driver.quit()关闭浏览器进程，避免资源占用

内容的提问来源于stack exchange，提问作者lignjoslav