使用XML包readHTMLTable()抓取合并单元格表格异常的解决请求

嘿，我太懂这种抓合并单元格表格踩坑的感觉了！readHTMLTable确实对rowspan/colspan这类合并单元格的支持很差，直接抓取肯定会出现结构混乱、内容缺失的问题。下面给你两个靠谱的解决思路：

方法一：用rvest包灵活处理合并单元格 #

rvest是更现代的网页抓取工具，配合手动解析单元格的合并属性，可以完美补全合并的内容：

首先安装并加载所需包：

install.packages(c("rvest", "dplyr"))
library(rvest)
library(dplyr)

然后编写抓取和处理逻辑：

# 目标网址
url <- "http://www.gaokao.com/e/20201106/5fa4f625338d0.shtml"

# 模拟浏览器请求头，避免被反爬
session <- session(url, user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36 QQBrowser/7.7.31732.400")
page <- read_html(session)

# 定位页面中的第一个表格
table_node <- page %>% html_nodes("table") %>% first()

# 提取表格的所有行
rows <- table_node %>% html_nodes("tr")

# 初始化变量：存储处理后的行数据、记录需要向下填充的rowspan内容
processed_rows <- list()
rowspan_queue <- list()

# 遍历每一行处理单元格
for (row in rows) {
  cells <- row %>% html_nodes("td, th")
  current_row <- list()
  col_index <- 1
  
  # 先处理上一行遗留的rowspan填充任务
  while (col_index <= length(rowspan_queue)) {
    if (rowspan_queue[[col_index]]$count > 0) {
      current_row[[col_index]] <- rowspan_queue[[col_index]]$value
      rowspan_queue[[col_index]]$count <- rowspan_queue[[col_index]]$count - 1
      col_index <- col_index + 1
    } else {
      # 移除已经完成填充的rowspan项
      rowspan_queue[[col_index]] <- NULL
    }
  }
  
  # 处理当前行的每个单元格
  for (cell in cells) {
    # 获取单元格文本并去除多余空格
    cell_text <- cell %>% html_text(trim = TRUE)
    
    # 获取rowspan和colspan属性，默认值为1（即没有合并）
    rowspan <- cell %>% html_attr("rowspan") %>% as.integer() %>% replace(is.na(.), 1)
    colspan <- cell %>% html_attr("colspan") %>% as.integer() %>% replace(is.na(.), 1)
    
    # 处理colspan：将单元格内容填充到对应的多列中
    for (i in 1:colspan) {
      current_row[[col_index]] <- cell_text
      col_index <- col_index + 1
    }
    
    # 处理rowspan：如果需要向下填充多行，记录到队列中
    if (rowspan > 1) {
      for (i in 0:(colspan-1)) {
        rowspan_queue[[col_index - i - 1]] <- list(value = cell_text, count = rowspan - 1)
      }
    }
  }
  
  # 将处理后的行加入列表
  processed_rows[[length(processed_rows)+1]] <- current_row
}

# 将处理后的行列表转换为标准数据框
max_cols <- max(sapply(processed_rows, length))
table_df <- bind_rows(lapply(processed_rows, function(row) {
  # 统一列数，空缺的位置补NA
  row <- c(row, rep(NA, max_cols - length(row)))
  as.data.frame(t(row), stringsAsFactors = FALSE)
}))

# 设置表格表头（假设第一行是表头）
header_row <- table_df[1,]
table_df <- table_df[-1,]
colnames(table_df) <- header_row

方法二：继续用XML包手动解析补全 #

如果你习惯用XML包，也可以通过手动解析节点属性来补全合并单元格的内容：

library(XML)
library(RCurl)
library(dplyr)

# 你的原始请求头设置
myHttpheader <- c( 
  "User-Agent"="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.48 Safari/537.36 QQBrowser/7.7.31732.400", 
  "Accept"="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 
  "Accept-Language"="en-us", 
  "Connection"="keep-alive", 
  "Accept-Charset"="GB2312,utf-8;q=0.7,*;q=0.7", 
  "Referer"="http://t.dianping.com/")
url = "http://www.gaokao.com/e/20201106/5fa4f625338d0.shtml"

# 抓取并转码网页内容
temp<-getURL(url,httpheader = myHttpheader,.encoding = "GB2312")
temp1<-iconv(temp,"GB2312","UTF-8")
doc<-htmlParse(temp1,asText = TRUE,encoding = "UTF-8")

# 获取第一个表格的所有行节点
rows <- getNodeSet(doc, "//table[1]/tr")

# 初始化变量
processed_rows <- list()
rowspan_queue <- list()

# 遍历每一行处理
for (row in rows) {
  cells <- getNodeSet(row, ".//td|.//th")
  current_row <- list()
  col_index <- 1
  
  # 处理上一行遗留的rowspan填充
  while (col_index <= length(rowspan_queue)) {
    if (rowspan_queue[[col_index]]$count > 0) {
      current_row[[col_index]] <- xmlValue(rowspan_queue[[col_index]]$node) %>% trimws()
      rowspan_queue[[col_index]]$count <- rowspan_queue[[col_index]]$count - 1
      col_index <- col_index + 1
    } else {
      rowspan_queue[[col_index]] <- NULL
    }
  }
  
  # 处理当前行的单元格
  for (cell in cells) {
    cell_text <- xmlValue(cell) %>% trimws()
    rowspan <- xmlGetAttr(cell, "rowspan", default = "1") %>% as.integer()
    colspan <- xmlGetAttr(cell, "colspan", default = "1") %>% as.integer()
    
    # 填充colspan列
    for (i in 1:colspan) {
      current_row[[col_index]] <- cell_text
      col_index <- col_index + 1
    }
    
    # 记录rowspan填充任务
    if (rowspan > 1) {
      for (i in 0:(colspan-1)) {
        rowspan_queue[[col_index - i - 1]] <- list(node = cell, count = rowspan - 1)
      }
    }
  }
  
  processed_rows[[length(processed_rows)+1]] <- current_row
}

# 转换为数据框并设置表头
max_cols <- max(sapply(processed_rows, length))
table_df <- bind_rows(lapply(processed_rows, function(row) {
  row <- c(row, rep(NA, max_cols - length(row)))
  as.data.frame(t(row), stringsAsFactors = FALSE)
}))

header_row <- table_df[1,]
table_df <- table_df[-1,]
colnames(table_df) <- header_row

核心思路都是手动解析单元格的rowspan和colspan属性，把合并的内容填充到对应的空缺位置，这样就能得到结构完整的表格数据了。

内容的提问来源于stack exchange，提问作者JexterSUFE