如何提升Nim异步请求的响应速度与请求配额，对标Python asyncio？

一、提升请求响应速度的方法 #

1. 复用HTTP连接池 #

Nim的asynchttpclient默认可能未启用连接复用或连接池配置不合理，会导致每次请求重建TCP连接增加耗时。可通过配置客户端参数开启长连接并设置合理的连接池大小：

import asyncdispatch, asynchttpclient, options

proc createOptimizedClient(): AsyncHttpClient =
  var settings = newHttpClientSettings()
  settings.enableKeepAlive = true
  settings.maxConnectionsPerHost = 10  # 根据目标服务器调整，大型站点可适当提高
  settings.connectionTimeout = 5000    # 单位：毫秒
  result = newAsyncHttpClient(settings)

使用时固定复用该客户端实例，避免每次请求创建新客户端。

2. 合理控制并发数 #

无限制并发会引发资源竞争或服务器限流，反而降低效率。使用AsyncSemaphore控制同时发起的请求数量：

import asyncdispatch, asynchttpclient, strutils

const MAX_CONCURRENT = 10
var sem = newAsyncSemaphore(MAX_CONCURRENT)

proc fetchUrl(url: string, client: AsyncHttpClient): Future[string] {.async.} =
  await sem.acquire()
  try:
    result = await client.getContent(url)
  finally:
    sem.release()

根据目标站点承受能力调整MAX_CONCURRENT，普通站点建议设5-10。

3. 优化DNS解析 #

默认DNS解析为同步阻塞，改用异步DNS解析库（如asyncdns）可减少等待时间：

import asyncdns, asynchttpclient

# 配置客户端使用异步DNS
var settings = newHttpClientSettings()
settings.dnsResolver = newAsyncDnsResolver()
var client = newAsyncHttpClient(settings)

二、避免Too Many Requests错误的方法 #

1. 实现带退避的重试机制 #

遇到429错误时，根据响应头Retry-After值延迟重试，或采用指数退避策略：

import asyncdispatch, asynchttpclient, httpcore, strutils

proc fetchWithRetry(url: string, client: AsyncHttpClient, maxRetries: int = 3): Future[string] {.async.} =
  var retries = 0
  while retries < maxRetries:
    try:
      let resp = await client.get(url)
      if resp.status == Http429:
        let retryAfter = resp.headers.getOrDefault("Retry-After").parseInt(5)
        await sleepAsync(retryAfter * 1000)
        retries.inc()
        continue
      result = await resp.body
      break
    except HttpClientError as e:
      await sleepAsync((2 ^ retries) * 1000)  # 指数退避延迟
      retries.inc()
  if retries == maxRetries:
    raise newException(HttpClientError, "Max retries exceeded for " & url)

2. 严格控制请求速率 #

用令牌桶算法限制每秒请求数，避免短时间内发送大量请求：

import asyncdispatch, times

const REQUESTS_PER_SECOND = 5
var tokens = REQUESTS_PER_SECOND
var lastRefill = getTime()

proc acquireToken(): Future[void] {.async.} =
  while true:
    let now = getTime()
    let elapsed = now - lastRefill
    if elapsed.inSeconds >= 1:
      tokens = REQUESTS_PER_SECOND
      lastRefill = now
    if tokens > 0:
      tokens.dec()
      break
    await sleepAsync(100)  # 等待令牌刷新

每个请求前调用await acquireToken()即可控制速率。

3. 模拟合法请求头 #

添加浏览器风格请求头，避免被识别为爬虫：

proc fetchWithHeaders(url: string, client: AsyncHttpClient): Future[string] {.async.} =
  var req = newHttpRequest(url)
  req.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
  req.headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
  req.headers["Accept-Language"] = "en-US,en;q=0.5"
  let resp = await client.send(req)
  result = await resp.body

三、通道与多客户端的正确用法 #

若用通道分发任务，可结合多客户端实例（每个对应独立连接池）处理不同批次请求：

import asyncdispatch, asynchttpclient, channels

type TaskMsg = tuple[url: string, resultChan: Channel[string]]

proc worker(client: AsyncHttpClient, taskChan: Channel[TaskMsg]) {.async.} =
  while true:
    let task = taskChan.recv()
    try:
      let content = await client.getContent(task.url)
      task.resultChan.send(content)
    except:
      task.resultChan.send("Error fetching " & task.url)

proc main() {.async.} =
  var taskChan: Channel[TaskMsg]
  taskChan.open()
  var resultChan: Channel[string]
  resultChan.open()

  # 创建3个客户端 worker
  for i in 0..<3:
    let client = createOptimizedClient()
    spawn worker(client, taskChan)

  # 发送任务示例
  let urls = ["https://example.com", "https://google.com"]
  for url in urls:
    taskChan.send((url, resultChan))

  # 接收结果
  for i in 0..<urls.len:
    echo await resultChan.receive()

  taskChan.close()
  resultChan.close()

waitFor main()

内容的提问来源于stack exchange，提问作者tenxsoydev