使用rvest爬取deutsche-biographie.de全量人物检索结果的实现方法

我希望使用rvest爬取https://www.deutsche-biographie.de/ 站点的数据。该网页顶部的输入框需要输入姓名后，才会返回所有该姓名或近似姓名的人物检索结果。

比如我输入姓名Meier后，使用如下代码爬取到了对应的检索结果：

library(rvest)
library(dplyr)

page = read_html(x = "https://www.deutsche-biographie.de/search?_csrf=45b6ee54-385e-4777-90bf-9067923e6a00&name=meier")
name = page %>% html_nodes(".media-heading a") %>% html_text()
information = page %>% html_nodes("div.media-body p") %>% html_text()
result = data.frame(name, information)
result$information <- result$information %>% trimws() %>% strsplit(split = ", \n") %>% lapply(trimws)
result <- tidyr::unnest_wider(result, information) %>%
  rename(years = 2, profession = 3) %>% 
  tidyr::extract(years, into = c("year_of_birth", "year_of_death"), regex = "^.*?(\\d{4}).*?\\-\\s(\\d{4})")

places <- page %>% html_nodes("li.treffer-liste-elem") %>% html_attr("data-orte") %>% strsplit(";")

result$place_of_birth <- lapply(places, function(x) x[grepl("@geburt$", x)]) %>% unlist()
result$place_of_death <- lapply(places, function(x) x[grepl("@tod$", x)]) %>% unlist()
result$place_of_activity <- lapply(places, function(x) x[grepl("@wirk$", x)])

result <- result %>% 
  tidyr::extract(place_of_birth, into = c("place_of_birth", "place_of_birth_coord"), regex = "^(.*?)@(.*?)@.*$") %>% 
  tidyr::extract(place_of_death, into = c("place_of_death", "place_of_death_coord"), regex = "^(.*?)@(.*?)@.*$")

result

本次使用的URL为"https://www.deutsche-biographie.de/search?_csrf=45b6ee54-385e-4777-90bf-9067923e6a00&name=meier"，其中name=meier是我手动输入的姓名。请问有没有无需指定单个姓名，即可获取全部姓名对应的全量检索结果的方法？

已验证解决方案 #

参照@QHarr的建议，新增for循环遍历所有结果页即可实现全量数据抓取，核心代码片段如下：

for (page_result in seq( from = 1, to = 2369 )) {
  link = paste0("https://www.deutsche-biographie.de/search?_csrf=8dc48621-c226-47b9-98b9-a9dfb2ab0ad8&name=*&number=",
                page_result)
...}

完整实现代码如下：

result_total = data.frame()

for (page_result in seq( from = 1, to = 2369 )) {
  link = paste0("https://www.deutsche-biographie.de/search?_csrf=8dc48621-c226-47b9-98b9-a9dfb2ab0ad8&name=*&number=",
                page_result)
  
  download.file(link, destfile = "scrapedpage.html", quiet=TRUE)
  #page = read_html("scrapedpage.html") #可取消注释避免超时报错：error in open.connection(x, "rb") : Timeout was reached
  page = read_html(link)
  name = page %>% html_nodes(".media-heading a") %>% html_text()
  information = page %>% html_nodes("div.media-body p") %>% html_text()
  result = data.frame(name, information)
  result$information <- result$information %>% trimws() %>% strsplit(split = ", \n") %>% lapply(trimws)
  result <- tidyr::unnest_wider(result, information) %>%
    rename(years = 2, profession = 3) %>% 
    tidyr::extract(years, into = c("year_of_birth", "year_of_death"), regex = "^.*?(\\d{4}).*?\\-\\s(\\d{4})")
  
  places <- page %>% html_nodes("li.treffer-liste-elem") %>% html_attr("data-orte") %>% strsplit(";")
  
  result$place_of_birth <- lapply(places, function(x) x[grepl("@geburt$", x)])
  result$place_of_death <- lapply(places, function(x) x[grepl("@tod$", x)])
  result$place_of_activity <- lapply(places, function(x) x[grepl("@wirk$", x)])
  
  result <- result %>% 
    tidyr::extract(place_of_birth, into = c("place_of_birth", "place_of_birth_coord"), regex = "^(.*?)@(.*?)@.*$") %>% 
    tidyr::extract(place_of_death, into = c("place_of_death", "place_of_death_coord"), regex = "^(.*?)@(.*?)@.*$")
  
  print(paste("Page:", page_result)) # 打印当前页码监控循环进度
  result_total <- rbind(result_total, result)
}

result_total <- apply(result_total,2,as.character)

注意：站点的csrf参数有有效期，运行代码前需要先访问站点获取最新的csrf值替换代码中的对应参数，否则会请求失败。爬取过程中建议添加随机延时，避免触发站点反爬机制。

内容的提问来源于stack exchange，提问作者natalieee