Got库异步请求408状态码全局传播问题求助

在循环中使用Got库批量请求1000个域名时，出现408状态码全局传播问题：只要某一个请求返回408响应，后续所有请求都会被标记为408。即使为每个请求单独设置超时，问题仍未解决。示例场景：解析1000个域名时，第一个请求成功，第二个返回408，后续所有请求立即返回408。

相关代码 #

Client类代码 #

const got = require('got');
const Logger = require('./logger');

let iconvLib = require('iconv-lite');

class Client
{
    constructor(url, data = {}, timeout = 6000, headers = {}) {
        this.url = url;
        this.data = data;
        this.timeout = timeout;
        this.customHeaders = headers;
        this.response = {};
        this.logger = new Logger();
    }

    getJson() {
        return JSON.parse(this.getBody());
    }

    getBody() {
        let response = this.getResponse();

        const contentType = response.headers['content-type'];
        const matches = contentType.match(/charset=([^;]+)/);
        let encoding = matches ? matches[1] : 'utf8';

        if(encoding === 'windows-1251') {
            encoding = 'win1251';
        }

        let body = iconvLib.decode(Buffer.from(response.body), encoding);

        //console.log(body);

        return body;
    }

    getHeaders() {
        return this.getResponse().headers;
    }

    getStatusText() {
        return this.getResponse().statusMessage;
    }

    getStatus() {
        return this.getResponse().statusCode;
    }

    getResponse() {
        return this.response ?? {};
    }

    async head() {
        const config = this.getConfig();
        if(Object.keys(this.data).length !== 0) {
            config.searchParams = this.data;
        }

        try {
            this.response = await got.head(this.url, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    async post() {
        const config = this.getConfig();
        if(Object.keys(this.data).length !== 0) {
            config.body = this.data;
        }

        try {
            this.response = await got.post(this.url, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    async get() {
        const config = this.getConfig();
        let params = '';
        if(Object.keys(this.data).length !== 0) {
            const sign = this.url.includes('?') ? '&' : '?';
            params += sign + this.serialize(this.data);
        }

        config.responseType = 'buffer';

        try {
            this.response = await got.get(this.url + params, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    handleErrors(e) {
        const response = e.response ?? {};
        const errorsStatusMap = {
            'ERR_TOO_MANY_REDIRECTS': 310,
            'ERR_NON_2XX_3XX_RESPONSE': 403,
            'ECONNREFUSED': 404,
            'ECONNRESET': 404,
            'ENOTFOUND': 404,
            'ERR_INVALID_URL': 404,
            'ETIMEDOUT': 408,
            'EAI_AGAIN': 408,
            'Z_BUF_ERROR': 507,
            'Z_DATA_ERROR': 507,
            'EPROTO': 526,
            'EHOSTUNREACH': 0,
            'HPE_INVALID_HEADER_TOKEN': 0,
            'ENETUNREACH': 0,
        };

        if(e.code) {
            response.statusCode = errorsStatusMap[e.code];
        }

        if(e.name) {
            response.statusText = e.name;
        }

        if(!response.body) {
            response.body = '';
        }

        if(!response.statusCode && response.statusCode !== 0) {
            response.statusCode = 0;
        }

        return response;
    }

    getConfig() {
        return {
            timeout: {
                request: this.timeout,
            },
            retry: {
                limit: 0,
                maxRetryAfter: undefined,
            },
            https: {
                rejectUnauthorized: false
            },
            headers: Object.assign({
                'Connection': 'close',
                'User-Agent': this.getUserAgent(),
                'Sec-Ch-Ua-Mobile': '?0',
                'Sec-Ch-Ua-Platform': 'Windows',
                'Sec-Fetch-Dest': 'document',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'none',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': 1,
                'Dnt': 1
            }, this.customHeaders)
        };
    }

    getUserAgent() {
        return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36';
    }

    serialize(obj, prefix) {
        let str = [],
            p;
        for (p in obj) {
            if (obj.hasOwnProperty(p)) {
                const k = prefix ? prefix + '[' + p + ']' : p,
                    v = obj[p];
                str.push((v !== null && typeof v === 'object') ?
                    this.serialize(v, k) :
                    encodeURIComponent(k) + '=' + encodeURIComponent(v));
            }
        }

        return str.join('&');
    }
}

module.exports = Client;

Client使用方式 #

async makeHttpsRequest(domain) {
    const client = new Client('https://' + domain);
    await client.get();

    return client;
}

解析器初始化代码 #

let parsers = [];
for(const domainItem of domainList) {
    parsers.push(new Parser(domainItem['domain'], domainItem['id'] ?? 0));
}

parsers = await Promise.all(parsers.map(parser => parser.init())); // <- 在此处调用makeHttpsRequest

解决思路 #

• 禁用连接池复用：Got默认复用TCP连接，单个请求超时可能污染连接池，导致后续请求复用无效连接直接返回408。在请求配置中添加独立的Agent，强制每次创建新连接：

  // 需要先引入http/https模块
  const http = require('http');
  const https = require('https');
  
  getConfig() {
      return {
          // ...其他配置
          agent: {
              http: new http.Agent({ keepAlive: false }),
              https: new https.Agent({ keepAlive: false })
          }
      };
  }
  

• 隔离错误响应对象：当前handleErrors直接修改e.response，而Got的错误对象可能存在共享引用，导致状态码传播。改为创建全新对象：

  handleErrors(e) {
      // 创建新对象，避免引用共享
      const response = e.response ? { ...e.response } : {};
      // ...后续逻辑不变
  }
  

• 限制并发请求数：一次性发起1000个请求易导致资源耗尽，触发批量超时。用p-limit等工具限制并发（比如20-50），避免请求堆积。
• 检查全局状态污染：确认iconvLib、Logger等依赖是否存在全局共享状态，确保每个Client实例完全独立。
• 升级Got版本：旧版本Got可能存在连接池或错误处理bug，升级到最新稳定版可解决已知问题。

内容的提问来源于stack exchange，提问作者Ronohx