循环处理9000条链接遇随机超时，求重试方案及优化建议

Great question—random timeouts are super common when processing thousands of web requests, and adding a retry mechanism is exactly the right approach. Let’s break down how to fix this and optimize your code for 9000 links.

1. Build a Retry-Aware GET Function #

First, let’s create a custom GET wrapper that automatically retries failed requests (either timeouts or non-200 status codes). This will handle those random flaky links without stopping your entire loop.

library(httr)
library(rvest)
library(stringr)

# Custom GET function with retries and timeout
retry_get <- function(url, max_tries = 3, timeout_sec = 120) {
  attempt <- 1
  while (attempt <= max_tries) {
    tryCatch({
      # Send the request with timeout
      response <- GET(url, timeout(timeout_sec))
      
      # Check if we got a successful response
      if (status_code(response) == 200) {
        return(response)
      } else {
        warning(sprintf("Attempt %d failed for %s (Status code: %d)", attempt, url, status_code(response)))
      }
    }, error = function(e) {
      # Catch timeout errors or other connection issues
      warning(sprintf("Attempt %d failed for %s (Error: %s)", attempt, url, e$message))
    })
    
    # Wait a bit before retrying to avoid overwhelming the server
    Sys.sleep(2)
    attempt <- attempt + 1
  }
  
  # If all retries fail, return NULL
  return(NULL)
}

2. Optimize Your Loop Logic #

Your original code uses rbind inside the loop, which gets really slow with 9000 iterations (every rbind copies the entire data frame). Instead, use a list to store results and combine them at the end. We’ll also add progress tracking so you can see where you’re at.

# Initialize storage (lists are way faster for iterative builds)
successful_results <- list()
failed_links <- list()

# Loop through each link
for (i in 1:nrow(final_links_bind)) {
  current_link <- final_links_bind[i, ]
  current_bp_id <- final_bp_bind[i, ]
  
  # Print progress to track where you are
  cat(sprintf("Processing link %d/%d: %s\n", i, nrow(final_links_bind), current_link))
  
  # Use our retry-enabled GET function
  response <- retry_get(current_link, max_tries = 3, timeout_sec = 120)
  
  if (!is.null(response)) {
    # Parse the successful response
    url_parse <- read_html(response) # Use the response directly, no need to re-fetch
    col_names <- url_parse %>%
      html_nodes("tr") %>%
      html_text() %>%
      str_remove_all("\\t|\\n|\\r")
    
    # Find the "využití" entry
    use_index <- grep("využití", col_names)
    
    if (length(use_index) > 0) {
      # Store the result in the list
      successful_results[[i]] <- data.frame(
        BP_ID = current_bp_id,
        method_selected = col_names[use_index],
        Links = current_link,
        stringsAsFactors = FALSE
      )
    } else {
      # No match found—add to failed links with reason
      failed_links[[i]] <- data.frame(
        BP_ID = current_bp_id,
        Links = current_link,
        reason = "No 'využití' entry found",
        stringsAsFactors = FALSE
      )
    }
  } else {
    # All retries failed—add to failed links
    failed_links[[i]] <- data.frame(
      BP_ID = current_bp_id,
      Links = current_link,
      reason = "All retries timed out or failed",
      stringsAsFactors = FALSE
    )
  }
  
  # Optional: Add a small delay between requests to be kind to the server
  Sys.sleep(0.5)
}

# Combine all results into data frames (fast, since we're doing it once)
WRITE_CSV_DATA <- do.call(rbind, successful_results)
not_working_link <- do.call(rbind, failed_links)

# Save to CSV
write.csv(WRITE_CSV_DATA, "successful_processing.csv", row.names = FALSE)
write.csv(not_working_link, "failed_links.csv", row.names = FALSE)

3. Bonus: Speed Up with Parallel Processing (Optional) #

If 9000 links are taking too long, you can use parallel processing to run multiple requests at once. Just be careful not to flood the server (start with 2-4 workers):

library(furrr)

# Set up parallel sessions (adjust workers based on your CPU)
plan(multisession, workers = 4)

# Create a combined data frame for easier mapping
combined_data <- cbind(
  Links = final_links_bind[, 1],
  BP_ID = final_bp_bind[, 1]
)

# Define a function to process a single row
process_single_link <- function(row) {
  response <- retry_get(row$Links, max_tries = 3, timeout_sec = 120)
  
  if (!is.null(response)) {
    url_parse <- read_html(response)
    col_names <- url_parse %>%
      html_nodes("tr") %>%
      html_text() %>%
      str_remove_all("\\t|\\n|\\r")
    
    use_index <- grep("využití", col_names)
    
    if (length(use_index) > 0) {
      return(data.frame(
        BP_ID = row$BP_ID,
        method_selected = col_names[use_index],
        Links = row$Links,
        reason = "Success",
        stringsAsFactors = FALSE
      ))
    } else {
      return(data.frame(
        BP_ID = row$BP_ID,
        method_selected = NA,
        Links = row$Links,
        reason = "No 'využití' entry found",
        stringsAsFactors = FALSE
      ))
    }
  } else {
    return(data.frame(
      BP_ID = row$BP_ID,
      method_selected = NA,
      Links = row$Links,
      reason = "All retries failed",
      stringsAsFactors = FALSE
    ))
  }
}

# Run parallel processing with progress bar
all_results <- future_map_dfr(combined_data, process_single_link, .progress = TRUE)

# Split into successful and failed
WRITE_CSV_DATA <- subset(all_results, reason == "Success")
not_working_link <- subset(all_results, reason != "Success")

Key Notes #

• Server Etiquette: Adding delays (Sys.sleep) helps avoid getting blocked by the target server. Always check the site’s robots.txt to make sure your scraping is allowed.
• Retry Limits: Don’t set max_tries too high (3-5 is usually enough)—if a link fails multiple times, it’s likely permanently broken or blocked.
• Error Handling: We added specific reasons for failures, so you can debug which links are truly broken vs. just flaky.

内容的提问来源于stack exchange，提问作者walle_eva