使用R实现动态URL网站多页面网页爬取的问题求助

原代码问题分析 #

• 未对每个URL执行read_html()解析，直接用字符串URL调用html_elements()，逻辑错误
• 函数定义不规范，未利用循环变量遍历URL列表
• 缺少请求延迟，容易触发PubMed反爬机制
• 内部函数get_abstract未正确整合到批量流程中

修复后的完整代码 #

library(rvest)
library(tidyverse)

# 生成所有页面URL，包含第1到407页
page_urls <- paste0("https://pubmed.ncbi.nlm.nih.gov/?term=((((((%E2%80%98Food%20Supply%E2%80%99%20(MeSH))%20OR%20%E2%80%98Food%20Storage%E2%80%99%20(MeSH))%20OR%20%E2%80%98Hunger%E2%80%99(MeSH)%20OR%20food%20security%20OR%20food%20insecurity%20OR%20household%20food%20security%20OR%20global%20food%20security)%20OR%20household%20food%20insecurity)))%20AND%20((%E2%80%98Prevalence%E2%80%99%20(MeSH))%20OR%20%E2%80%98Cross-Sectional%20Studies%E2%80%99%20(MeSH)%20OR%20cross-sectional%20study%20OR%20Prevalence%20Studies%20OR%20prevalence%20study%20OR%20Cross-Sectional%20Analyses%20OR%20CrossSectional%20Analysis%20OR%20Cross%20Sectional%20Analysis%20OR%20Cross%20Sectional%20Analyses)&filter=lang.english&filter=lang.portuguese&page=", 1:407)

# 定义单页列表数据提取函数
extract_page_data <- function(url) {
  # 添加随机延迟，规避反爬
  Sys.sleep(runif(1, 1, 3))
  cat("正在爬取页面:", url, "\n")
  
  # 解析目标页面
  page <- read_html(url)
  
  # 提取单页内的文献信息
  page %>%
    html_elements(".docsum-content") %>%
    map_dfr(~ tibble(
      title = .x %>% html_element(".docsum-title") %>% html_text2(),
      authors = .x %>% html_element(".full-authors") %>% html_text2(),
      PMID = .x %>% html_element(".docsum-pmid") %>% html_text2(),
      synopsis = .x %>% html_element(".full-view-snippet") %>% html_text2(),
      link = .x %>% html_element(".docsum-title") %>% html_attr("href") %>% str_c("https://pubmed.ncbi.nlm.nih.gov", .)
    ))
}

# 定义摘要提取函数，包含错误捕获
extract_abstract <- function(link) {
  Sys.sleep(runif(1, 0.5, 2))
  cat("正在爬取摘要:", link, "\n")
  
  tryCatch({
    read_html(link) %>%
      html_elements(".abstract-content.selected") %>%
      html_text2() %>%
      str_squish()
  }, error = function(e) {
    warning("爬取摘要失败:", link, " | 错误信息:", e$message)
    return(NA_character_)
  })
}

# 批量爬取所有页面的列表数据
all_pubmed_data <- map_dfr(page_urls, extract_page_data)

# 批量提取每个文献的摘要
all_pubmed_data <- all_pubmed_data %>%
  mutate(abstract = map_chr(link, extract_abstract))

# 查看结果示例
head(all_pubmed_data)

关键修复点说明 #

• URL遍历逻辑：用map_dfr遍历生成的page_urls向量，每个URL单独解析处理
• 反爬处理：添加随机延迟，降低被PubMed封禁IP的风险
• 错误容错：给摘要提取函数添加tryCatch，避免单个链接爬取失败导致整个流程中断
• 函数拆分：将单页提取和摘要提取拆分为独立函数，逻辑更清晰易维护
• 完整数据覆盖：URL生成包含第1到407页，确保获取全部目标文献数据

内容的提问来源于stack exchange，提问作者Ileeo