大并发Http/2请求报错ERR_HTTP2_STREAM_ERROR的原因与解决方案咨询

问题背景 #

构建HTTP/2服务器时，尝试向服务器发送5万+请求，发送到一定数量后出现报错。在客户端请求间添加0.5ms或0.05ms延迟时，成功请求数随延迟增加，但不需要任何延迟，希望完成更大规模负载测试。

报错信息 #

客户端报错 #

error : {"Error":{"code":"ERR_HTTP2_STREAM_ERROR"},"index":29498}
error : {"Error":{"code":"ERR_HTTP2_STREAM_ERROR"},"index":29499}
...

服务器端报错 #

{"msg":"Error occured Error [ERR_HTTP2_STREAM_ERROR]: Stream closed with error code NGHTTP2_INTERNAL_ERROR","streamId":44703,"time":1.083,"total time":5.414000000000001,"index":"22351"}
error : {"msg":"Error occured Error [ERR_HTTP2_STREAM_ERROR]: Stream closed with error code NGHTTP2_INTERNAL_ERROR","streamId":44705,"time":1.083,"total time":6.497000000000001,"index":"22352"}
...

历史报错 #

error=Error [ERR_HTTP2_STREAM_ERROR]: Stream closed with error code NGHTTP2_INTERNAL_ERROR

原因分析 #

• HTTP/2流并发限制：Node.js的HTTP/2客户端、服务器默认都有并发流上限，短时间内发起大量请求会超出阈值，触发流关闭机制。
• 服务器事件循环阻塞：服务器中同步日志输出、Redis异步操作积压，会阻塞事件循环，导致无法及时处理新请求，引发流错误。
• TCP缓冲区溢出：无延迟发送大量请求会填满TCP发送缓冲区，数据无法及时传输，触发流异常。
• 客户端请求调度不合理：循环+setTimeout的方式本质还是短时间内发起大量请求，未做有效流控制，超出HTTP/2窗口限制。
• Redis连接池耗尽：大量请求同时触发Redis操作，若连接池配置过小，会导致无法获取连接，进而引发流错误。

解决方案 #

客户端优化 #

1. 调整HTTP/2连接参数：提升并发流和内存限制，适配大流量场景：

const client = http2.connect('https://localhost:5000', {
  ca: fs.readFileSync('localhost-cert.pem'),
  maxSessionMemory: 100 * 1024 * 1024, // 100MB会话内存
  maxConcurrentStreams: 1000 // 并发流上限，需和服务器匹配
});

2. 实现请求并发控制：用队列限制同时发送的请求数，避免瞬间压爆连接，可使用p-limit库：

const limit = require('p-limit')(500); // 控制500并发

client.on('connect', () => {
  const promises = [];
  for (let i = 0; i < 50000; i++) {
    promises.push(limit(() => sendRequest(i)));
  }
  Promise.allSettled(promises).then(() => client.close());
});

3. 重构请求逻辑：将sendRequest改为Promise形式，便于并发控制，去掉无效的setTimeout调度。

服务器优化 #

1. 调整HTTP/2服务器配置：同步提升并发流和内存限制：

const server = http2.createSecureServer({
  key: fs.readFileSync('localhost-privkey.pem'),
  cert: fs.readFileSync('localhost-cert.pem'),
  maxSessionMemory: 200 * 1024 * 1024,
  maxConcurrentStreams: 1000
});

2. 优化Redis连接池：配置足够大的连接池，避免连接耗尽：

// 示例Redis客户端配置（需根据实际Redis库调整）
const RedisClient = require('redis').createClient({
  url: 'redis://localhost:6379',
  socket: {
    reconnectStrategy: (retries) => Math.min(retries * 50, 500)
  },
  database: 0,
  pool: { max: 1000 } // 连接池最大连接数
});

3. 减少事件循环阻塞：将同步日志操作改为异步，避免阻塞请求处理：

stream.on('data', (chunk) => {
  // 异步处理日志
  process.nextTick(() => {
    logger.info(JSON.stringify({ "chunk": chunk }));
  });
  data += chunk;
});

4. 提前发送响应头：收到请求后先发送响应头，提升流复用效率，不要等到Redis操作完成：

stream.on('end', () => {
  // 先发送响应头
  stream.respond({ ':status': 200, index: index });
  // 再处理Redis逻辑
  try {
    const payload = data === '' ? '{}' : JSON.parse(data);
    RedisClient.setKey(payload.key, payload.value).then(response => {
      stream.end(JSON.stringify({
        "msg": "Redis key set success",
        "index": index,
        "streamId": stream.id
      }));
    }).catch(error => {
      stream.end(JSON.stringify({
        "msg": "Redis key set failure Error",
        "index": index,
        "streamId": stream.id
      }));
    });
  } catch (error) {
    stream.end(JSON.stringify({
      "msg": "Redis key set failure",
      "index": index,
      "streamId": stream.id
    }));
  }
});

系统层面优化 #

• 调整TCP参数：Linux系统下修改net.core.wmem_max、net.core.rmem_max等参数，提升TCP缓冲区大小，增强网络吞吐能力：

# 临时生效
sysctl -w net.core.wmem_max=16777216
sysctl -w net.core.rmem_max=16777216

# 永久生效，写入/etc/sysctl.conf
echo "net.core.wmem_max=16777216" >> /etc/sysctl.conf
echo "net.core.rmem_max=16777216" >> /etc/sysctl.conf
sysctl -p

• 服务器集群扩展：若单节点资源瓶颈无法解决，可采用Node.js集群模式或多服务器负载均衡，分散请求压力。

优化后客户端代码 #

const http2 = require('node:http2');
const fs = require('node:fs');
const logger = require('gk-logger')();
const limit = require('p-limit')(500);

const generateRandomWord = (length = 10) => {
  const alphabetList = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
  let randomWord = '';
  for (let i = 0; i < length; i++) {
    const randomIndex = Math.floor(Math.random() * alphabetList.length);
    randomWord += alphabetList[randomIndex];
  }
  return randomWord;
};

const client = http2.connect('https://localhost:5000', {
  ca: fs.readFileSync('localhost-cert.pem'),
  maxSessionMemory: 100 * 1024 * 1024,
  maxConcurrentStreams: 1000
});

client.on('error', (err) => console.error("client", err));

const getTime = (startTime) => {
  return (Date.now() - startTime) / 1000;
};

const sendRequest = (i) => {
  return new Promise((resolve, reject) => {
    const startTime = Date.now();
    logger.info(`Request is being send ${i}....`);

    const req = client.request({
      ':path': '/',
      ':method': 'POST',
      'rid': i
    });

    const payload = {
      type: 'POST Request',
      rid: i,
      key: generateRandomWord(10),
      value: generateRandomWord(20)
    };
    req.write(JSON.stringify(payload), 'utf-8');
    req.end();

    req.on('error', (err) => {
      logger.error(JSON.stringify({
        "Error": err,
        "index": i
      }));
      reject(err);
    });

    req.on('response', (headers) => {
      logger.info(JSON.stringify({
        "Response headers": headers,
        "index": i
      }));
    });

    let data = '';
    req.setEncoding('utf-8');
    req.on('data', (chunk) => {
      data += chunk;
    });

    req.on('end', () => {
      const timeRequired = getTime(startTime);
      logger.info(JSON.stringify({
        "Client Data Received": data,
        "Client Time Required": timeRequired
      }));
      resolve();
    });

    logger.info(`Reach at the end of request`);
  });
};

client.on('connect', () => {
  console.log("Connected to server");
  const promises = [];
  for (let i = 0; i < 50000; i++) {
    promises.push(limit(() => sendRequest(i)));
  }
  Promise.allSettled(promises).then(() => {
    console.log("All requests processed, closing connection");
    client.close();
  });
});

client.on('close', () => {
  console.log("TCP connection closed");
});

优化后服务器代码 #

const http2 = require('node:http2');
const fs = require('node:fs');
const logger = require('gk-logger')();
const RedisClient = require('./redis/redisClient');

const server = http2.createSecureServer({
  key: fs.readFileSync('localhost-privkey.pem'),
  cert: fs.readFileSync('localhost-cert.pem'),
  maxSessionMemory: 200 * 1024 * 1024,
  maxConcurrentStreams: 1000
});

let total_time = 0;
const getTime = (startTime) => {
  return (Date.now() - startTime) / 1000;
};

server.on('stream', (stream, headers) => {
  const startTime = Date.now();
  const index = headers['rid'] ?? 'null';
  const method = headers[':method'];

  if (method === 'GET') {
    stream.end(JSON.stringify({
      "Method": "Get Request",
      index: index
    }));
    return;
  }
  logger.info(`request received on server ${index}`);
  let data = '';
  stream.setEncoding('utf-8');
  stream.on('data', (chunk) => {
    process.nextTick(() => {
      logger.info(JSON.stringify({ "chunk": chunk }));
    });
    data += chunk;
  });

  stream.on('aborted', () => {
    const timeRequired = getTime(startTime);
    total_time += timeRequired;
    logger.error({
      msg: 'Stream Aborted',
      streamId: stream.id,
      time: timeRequired,
      "total time": total_time,
      "index": index
    });
  });

  stream.on('error', (err) => {
    const timeRequired = getTime(startTime);
    total_time += timeRequired;
    logger.error(JSON.stringify({
      msg: `Error occured ${err}`,
      streamId: stream.id,
      time: timeRequired,
      "total time": total_time,
      "index": index
    }));
  });

  stream.on('end', () => {
    logger.info(JSON.stringify({ "Data received @server": data }));
    stream.respond({ ':status': 200, index: index });
    try {
      const payload = data === '' ? '{}' : JSON.parse(data);
      RedisClient.setKey(payload.key, payload.value).then(response => {
        logger.info(JSON.stringify(`Redis key set successfully res:${response} key:${payload.key} Value:${payload.value} index:${index}`));
        stream.end(JSON.stringify({
          "msg": "Redis key set success",
          "index": index,
          "streamId": stream.id
        }));
      }).catch(error => {
        logger.error(JSON.stringify({ "Error while parsing or setting redis key": error }));
        stream.end(JSON.stringify({
          "msg": "Redis key set failure Error",
          "index": index,
          "streamId": stream.id
        }));
      });
    } catch (error) {
      logger.error(JSON.stringify({ "Error while parsing or setting redis key": error }));
      stream.end(JSON.stringify({
        "msg": "Redis key set failure",
        "index": index,
        "streamId": stream.id
      }));
    }
  });
});

const port = 5000;
server.listen(port, () => {
  console.log(`Server running https://localhost:${port}`);
});

内容的提问来源于stack exchange，提问作者Shubham Thorat