如何用Python抓取带展开按钮的网站表格数据？

我需要从https://bezerocarbon.com/listings构建数据库，网站需填写随机个人信息解锁完整表格。使用BeautifulSoup提取数据时，无法获取展开（+）按钮下的项目详情，现有代码仅能抓取表层表格数据，缺失展开后的详情内容。

现有代码 #

import requests
from bs4 import BeautifulSoup
import pandas as pd

url = 'https://bezerocarbon.com/listings/'
result = requests.get(url)
content = result.text
soup = BeautifulSoup(content,'lxml')

print(soup.prettify())
print(result) #received Response 200

table = soup.find('table', role='table')

#extract column names
headers = table.find_all('th')
titles = []
for i in headers:
    title = i.text
    titles.append(title)

#creating pandas dataframe
df = pd.DataFrame(columns=titles)

#extracting row details
#obtain all tr tags beccause of data are in rows, then obtain the details
rows = table.find_all('tr')
for i in rows[1:]: #this skips the first row which is the header
    tdata = i.find_all('td')
    #row = [tr.text for tr in data]
    row = [tr.text for tr in tdata]
    l = len(df)
    df.loc[l]=row

print(df)

问题原因 #

展开内容是动态加载的：点击"+"按钮时，页面会通过AJAX请求获取对应行的详情数据并渲染到页面上。直接用requests.get只能拿到页面初始化的静态HTML，不包含这些动态加载的内容，所以BeautifulSoup无法抓取到。

解决方案1：用Selenium模拟浏览器操作 #

通过模拟浏览器打开页面、填写解锁信息、点击所有展开按钮，再提取完整数据：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time

# 初始化Chrome浏览器（需提前下载对应版本的chromedriver）
driver = webdriver.Chrome()
driver.get('https://bezerocarbon.com/listings/')

# 填写随机信息解锁表格
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, 'name'))).send_keys('John Doe')
driver.find_element(By.NAME, 'email').send_keys('john@example.com')
driver.find_element(By.NAME, 'company').send_keys('Test Corp')
driver.find_element(By.CSS_SELECTOR, 'button[type="submit"]').click()

# 等待表格加载完成
time.sleep(3)

# 遍历点击所有展开按钮
expand_buttons = driver.find_elements(By.CSS_SELECTOR, 'button[aria-label="Expand row"]')
for btn in expand_buttons:
    driver.execute_script("arguments[0].click();", btn)
    time.sleep(1) # 等待详情加载

# 提取完整表格数据
table = driver.find_element(By.CSS_SELECTOR, 'table[role="table"]')
headers = [th.text.strip() for th in table.find_elements(By.TAG_NAME, 'th')]
rows = table.find_elements(By.TAG_NAME, 'tr')
final_data = []
current_main_row = []

for row in rows:
    row_classes = row.get_attribute('class')
    # 识别主行和展开行
    if 'MuiTableRow-root' in row_classes and not 'MuiTableRow-hover' in row_classes:
        if current_main_row:
            final_data.append(current_main_row)
        current_main_row = [td.text.strip() for td in row.find_elements(By.TAG_NAME, 'td')]
    elif 'MuiTableRow-hover' in row_classes:
        # 将展开行的详情合并到主行
        detail_cells = [td.text.strip() for td in row.find_elements(By.TAG_NAME, 'td')]
        current_main_row.extend(detail_cells)

# 处理最后一行数据
if current_main_row:
    final_data.append(current_main_row)

# 构建DataFrame（需根据实际展开后的内容调整列名）
df = pd.DataFrame(final_data, columns=headers + ['项目详情1', '项目详情2', '项目详情3'])
print(df)

driver.quit()

解决方案2：直接调用AJAX接口（更高效） #

1. 打开浏览器开发者工具（F12），切换到Network标签
2. 点击某一行的"+"按钮，观察XHR请求，找到获取详情的API接口（格式类似https://bezerocarbon.com/api/get-listing-details?id=xxx）
3. 提取所有主行的ID，批量请求接口获取详情，再和主行数据合并

示例代码思路：

import requests
from bs4 import BeautifulSoup
import pandas as pd

# 先获取主行数据及每行ID
url = 'https://bezerocarbon.com/listings/'
result = requests.get(url)
soup = BeautifulSoup(result.text, 'lxml')

table = soup.find('table', role='table')
headers = [th.text.strip() for th in table.find_all('th')]
rows = table.find_all('tr')[1:]
main_data = []
item_ids = []

for row in rows:
    tds = row.find_all('td')
    main_row = [td.text.strip() for td in tds]
    main_data.append(main_row)
    # 从行属性中提取ID（需根据页面实际结构调整）
    item_id = row.get('data-id')
    item_ids.append(item_id)

# 批量请求详情接口
details_data = []
for item_id in item_ids:
    detail_url = f'https://bezerocarbon.com/api/get-listing-details?id={item_id}'
    # 若需要请求头验证，可添加headers参数
    detail_result = requests.get(detail_url)
    details = detail_result.json()
    details_data.append(list(details.values()))

# 合并主数据和详情数据
final_data = [main + detail for main, detail in zip(main_data, details_data)]
df = pd.DataFrame(final_data, columns=headers + ['详情字段1', '详情字段2'])
print(df)

注意事项 #

• 使用Selenium时，需确保浏览器驱动版本与浏览器版本匹配
• AJAX接口的URL和参数需根据页面实际网络请求确认，部分接口可能需要携带Cookie或Referer请求头
• 注意控制请求频率，避免触发网站反爬机制

内容的提问来源于stack exchange，提问作者Timmy