求助：使用Python BeautifulSoup与Selenium抓取HMP动态表格File UUID列失败

使用Python的BeautifulSoup和Selenium从人类微生物组项目（HMP）网站抓取动态表格数据时，其他列均可正常获取，但File UUID列始终无法抓取到。已通过浏览器元素确认该列存在于id为files-table的表格下，预期能获取完整表格数据，但实际仅缺失目标列。

原代码 #

from bs4 import BeautifulSoup
from selenium import webdriver
import time

import numpy as np
import pandas as pd

# 连接HMP网站并解析表格信息
url = 'https://portal.hmpdacc.org/query/f?query=file.matrix_type%20in%20%5B%22wgs_community%22,%2216s_community%22%5D%20and%20sample.body_site%20in%20%5B%22feces%22%5D&filters=%7B%22op%22:%22and%22,%22content%22:%5B%7B%22op%22:%22in%22,%22content%22:%7B%22field%22:%22file.matrix_type%22,%22value%22:%5B%22wgs_community%22,%2216s_community%22%5D%7D%7D,%7B%22op%22:%22in%22,%22content%22:%7B%22field%22:%22sample.body_site%22,%22value%22:%5B%22feces%22%5D%7D%7D%5D%7D#:~:text=Samples%20(3%2C452)-,Files%20(5%2C181),-files'
browser = webdriver.Chrome()
browser.get(url)
time.sleep(3)
html = browser.page_source
hmp_parsed_page = BeautifulSoup(html, "lxml")
hmp_files_table = hmp_parsed_page.find('table', id='files-table')

# 获取表格列标题
hmp_metadata_fields = []
for th in hmp_files_table.find_all('th'):
    col_header = th.text
    hmp_metadata_fields.append(col_header)

# 创建DataFrame存储数据
hmp_metadata_df = pd.DataFrame(columns = hmp_metadata_fields)

# 逐行提取数据并添加到DataFrame
for tr in hmp_files_table.find_all('tr')[1:]:
    row_data = tr.find_all('td')
    row = [data_point.text for data_point in row_data]
    hmp_metadata_df.loc[len(hmp_metadata_df.index)] = row

# 删除不需要的列
hmp_metadata_df = hmp_metadata_df.drop(hmp_metadata_df.columns[[0,1]], axis = 1)

# 添加数据源标识
hmp_metadata_df['Data Source'] = 'HMP'
print(hmp_metadata_df)

# 关闭浏览器连接
browser.close()
browser.quit()

问题分析 #

1. 页面渲染不充分：原代码使用固定time.sleep(3)等待页面加载，但HMP网站的表格基于Angular动态渲染，File UUID列的内容可能需要更长时间或特定触发才能完全加载，固定等待时间无法保证内容渲染完成。
2. 文本提取方式不当：直接通过td.text提取内容时，若File UUID列的DOM结构存在嵌套（如嵌套span或其他元素），可能无法正确获取到文本内容。

解决方案 #

改用Selenium的显式等待确保表格元素加载完成，直接通过Selenium定位表格行与单元格，避免依赖BeautifulSoup解析可能未完全渲染的源码：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

url = 'https://portal.hmpdacc.org/query/f?query=file.matrix_type%20in%20%5B%22wgs_community%22,%2216s_community%22%5D%20and%20sample.body_site%20in%20%5B%22feces%22%5D&filters=%7B%22op%22:%22and%22,%22content%22:%5B%7B%22op%22:%22in%22,%22content%22:%7B%22field%22:%22file.matrix_type%22,%22value%22:%5B%22wgs_community%22,%2216s_community%22%5D%7D%7D,%7B%22op%22:%22in%22,%22content%22:%7B%22field%22:%22sample.body_site%22,%22value%22:%5B%22feces%22%5D%7D%7D%5D%7D#:~:text=Samples%20(3%2C452)-,Files%20(5%2C181),-files'

browser = webdriver.Chrome()
browser.get(url)

# 显式等待表格加载完成，最长等待10秒
wait = WebDriverWait(browser, 10)
files_table = wait.until(EC.presence_of_element_located((By.ID, 'files-table')))

# 获取列标题
headers = [th.text.strip() for th in files_table.find_elements(By.TAG_NAME, 'th')]

# 获取所有行数据
rows = files_table.find_elements(By.TAG_NAME, 'tr')[1:]  # 跳过表头行
data = []
for row in rows:
    # 获取每行的单元格文本，确保提取到嵌套元素的内容
    cells = [cell.text.strip() for cell in row.find_elements(By.TAG_NAME, 'td')]
    data.append(cells)

# 构建DataFrame
hmp_metadata_df = pd.DataFrame(data, columns=headers)

# 后续处理逻辑不变
hmp_metadata_df = hmp_metadata_df.drop(hmp_metadata_df.columns[[0,1]], axis=1)
hmp_metadata_df['Data Source'] = 'HMP'
print(hmp_metadata_df)

# 关闭浏览器
browser.quit()

额外优化建议 #

• 若表格存在分页，需添加分页遍历逻辑，逐页加载并抓取数据
• 可尝试使用browser.execute_script()直接获取页面的Angular数据模型，绕过DOM解析，效率更高：

  # 执行JS获取表格数据（需确认页面Angular变量名，示例为推测）
  table_data = browser.execute_script("return tsc.tableParams.data;")
  hmp_metadata_df = pd.DataFrame(table_data)
  

内容的提问来源于stack exchange，提问作者Nathaniel Landi