如何从非结构化分页HTML站点抓取宿主易感性与非易感性相关数据

1 单页面目标数据提取 #

你已经拿到了h4和li的文本向量，只需要通过关键词定位两个目标标题的位置，截取对应区间的宿主列表即可，核心逻辑如下：

• 用which()匹配Susceptible host species和Insusceptible host species两个标题的索引位置
• 两个索引中间的元素为易感宿主列表，非易感索引之后到下一个h4标题前的元素为非易感宿主列表
  对应代码片段：

# 接你之前的代码，得到清理后的文本向量text_vec
text_vec <- pvo %>%
  html_elements("h4, li") %>%
  html_text() %>%
  str_replace_all("[\n]" , "") %>%
  str_squish() # 额外加一步去除首尾空白

# 定位两个目标标题的位置
sus_idx <- which(str_detect(text_vec, "Susceptible host species"))
insus_idx <- which(str_detect(text_vec, "Insusceptible host species"))

# 提取易感宿主：sus_idx下一个元素开始，到insus_idx前一个元素结束
sus_hosts <- text_vec[(sus_idx+1):(insus_idx-1)]
# 提取非易感宿主：insus_idx下一个元素开始，到下一个h4标题前结束（h4标题通常包含species/Description等大写开头的特征词，可根据实际输出调整规则）
next_h4_idx <- which(str_detect(text_vec, "^[A-Z].*species|^Description|^Diagnosis") & seq_along(text_vec) > insus_idx)[1]
insus_hosts <- ifelse(is.na(next_h4_idx), text_vec[(insus_idx+1):length(text_vec)], text_vec[(insus_idx+1):(next_h4_idx-1)])

2 批量爬取所有物种页面 #

首先爬取物种列表页的所有详情页链接，再循环爬取每个页面，注意添加爬取延时避免触发站点反爬：

# 1. 提取所有详情页链接
index_url <- "http://bio-mirror.im.ac.cn/mirrors/pvo/vide/sppindex.htm"
base_url <- "http://bio-mirror.im.ac.cn/mirrors/pvo/vide/"

all_links <- read_html(index_url) %>%
  html_elements("a") %>%
  html_attr("href") %>%
  str_subset("descr\\d+\\.htm") %>% # 匹配详情页链接格式
  paste0(base_url, .) # 拼接完整URL

# 2. 批量爬取函数
scrape_host_data <- function(url) {
  Sys.sleep(1) # 每次爬取间隔1秒，可自行调整
  tryCatch({
    page <- read_html(url)
    text_vec <- page %>%
      html_elements("h4, li") %>%
      html_text() %>%
      str_replace_all("[\n]" , "") %>%
      str_squish()
    # 提取物种名，可根据页面实际结构调整规则
    species_name <- page %>% html_elements("h2") %>% html_text() %>% first() %>% str_squish()
    # 定位宿主列表
    sus_idx <- which(str_detect(text_vec, "Susceptible host species"))
    insus_idx <- which(str_detect(text_vec, "Insusceptible host species"))
    if(length(sus_idx) ==0 || length(insus_idx)==0) {
      return(tibble(species = species_name, susceptible = NA_character_, insusceptible = NA_character_))
    }
    sus_hosts <- text_vec[(sus_idx+1):(insus_idx-1)] %>% paste0(collapse = "; ")
    next_h4_idx <- which(str_detect(text_vec, "^[A-Z].*species|^Description|^Diagnosis") & seq_along(text_vec) > insus_idx)[1]
    insus_hosts <- ifelse(is.na(next_h4_idx), 
                          text_vec[(insus_idx+1):length(text_vec)] %>% paste0(collapse = "; "),
                          text_vec[(insus_idx+1):(next_h4_idx-1)] %>% paste0(collapse = "; "))
    return(tibble(species = species_name, susceptible = sus_hosts, insusceptible = insus_hosts))
  }, error = function(e) {
    message(paste0("爬取失败：", url))
    return(tibble(species = NA_character_, susceptible = NA_character_, insusceptible = NA_character_))
  })
}

# 3. 批量执行爬取
library(purrr)
all_data <- map_dfr(all_links, scrape_host_data)

3 数据存储 #

爬取完成后直接导出为csv即可：

write.csv(all_data, "pvo_host_data.csv", row.names = F, na = "")

内容的提问来源于stack exchange，提问作者Dieter Kahl