如何在R中使用curl或RCurl下载并处理GISAID的JSON.xz格式文件

依赖安装 #

先安装所需依赖包：

install.packages(c("curl", "jsonlite"))

核心函数实现（curl包方案，推荐） #

该方案完全匹配你原bash脚本的逻辑，支持自定义参数、下载重试，默认保留xz压缩格式适配你后续读取需求：

download_gisaid <- function(
    gisaid_username,
    gisaid_password,
    gisaid_feed,
    save_dir = "results",
    decompress = FALSE, # 默认不做解压，保留.xz格式
    retry_times = 4 # 与原bash脚本重试次数一致
) {
  # 检查并创建保存目录
  if (!dir.exists(save_dir)) {
    dir.create(save_dir, recursive = TRUE, showWarnings = FALSE)
  }

  # 构造下载链接
  download_url <- sprintf(
    "https://www.epicov.org/epi3/3p/%s/export/provision.json.xz",
    gisaid_feed
  )

  # 配置curl认证与传输参数
  h <- curl::new_handle(
    username = gisaid_username,
    password = gisaid_password,
    failonerror = TRUE
  )
  h <- curl::handle_setopt(h, low_speed_time = 30, low_speed_limit = 1024)

  # 执行下载逻辑
  if (!decompress) {
    # 直接保存xz格式文件
    save_path <- file.path(save_dir, "provision.json.xz")
    for (i in seq_len(retry_times)) {
      tryCatch({
        curl::curl_download(download_url, destfile = save_path, handle = h)
        message("文件下载完成，保存路径：", save_path)
        break
      }, error = function(e) {
        if (i == retry_times) {
          stop("下载重试", retry_times, "次后失败，错误信息：", e$message)
        }
        message("第", i, "次下载失败，2秒后重试...")
        Sys.sleep(2)
      })
    }
    return(invisible(save_path))
  } else {
    # 边下载边解压，输出json文件
    save_path <- file.path(save_dir, "gisaid.json")
    con <- curl::curl(download_url, handle = h)
    on.exit(close(con))
    xz_con <- xzcon(con, open = "rb")
    on.exit(close(xz_con), add = TRUE)
    writeBin(readBin(xz_con, what = "raw", n = 1e8), save_path)
    message("文件下载并解压完成，保存路径：", save_path)
    return(invisible(save_path))
  }
}

使用示例 #

# 保存为xz格式，适配后续读取需求
download_gisaid(
  gisaid_username = "你的GISAID用户名",
  gisaid_password = "你的GISAID密码",
  gisaid_feed = "你的GISAID feed ID",
  save_dir = "./data/GISAID_json"
)

# 如需直接解压为json文件，添加参数decompress = TRUE即可

后续读取注意事项 #

你现有读取代码中用gzfile读取xz格式文件是错误的，需要替换为xzfile才能正常读取：

library(jsonlite)
GISAID_json <- jsonlite::stream_in(xzfile("./data/GISAID_json/provision.json.xz"))

RCurl包备选实现 #

如果你的运行环境仅支持RCurl包，可以使用以下版本：

# 先安装依赖
# install.packages("RCurl")
download_gisaid_rcurl <- function(
    gisaid_username,
    gisaid_password,
    gisaid_feed,
    save_dir = "results",
    decompress = FALSE,
    retry_times = 4
) {
  if (!dir.exists(save_dir)) dir.create(save_dir, recursive = TRUE)
  download_url <- sprintf(
    "https://www.epicov.org/epi3/3p/%s/export/provision.json.xz",
    gisaid_feed
  )
  save_path <- if (decompress) file.path(save_dir, "gisaid.json") else file.path(save_dir, "provision.json.xz")
  
  for (i in seq_len(retry_times)) {
    tryCatch({
      content <- RCurl::getBinaryURL(
        download_url,
        userpwd = paste0(gisaid_username, ":", gisaid_password),
        followlocation = TRUE
      )
      if (decompress) content <- memDecompress(content, type = "xz")
      writeBin(content, save_path)
      message("下载完成，保存路径：", save_path)
      break
    }, error = function(e) {
      if (i == retry_times) stop("重试", retry_times, "次后失败：", e$message)
      message("第", i, "次下载失败，2秒后重试...")
      Sys.sleep(2)
    })
  }
  return(invisible(save_path))
}

内容的提问来源于stack exchange，提问作者Tom Wenseleers