Docker环境下NodeJS API高负载超时问题排查与解决

我有一个运行在集群模式下的Node.js API，使用PostgreSQL连接池处理I/O操作。本地运行一切正常，但用Gatling做压力测试时，测试进行到一半会出现大量超时。我已经验证过查询性能没问题，问题应该出在配置或代码的错误上。

docker-compose.yml 配置 #

version: '3.4'

services:
  app1: &app
    container_name: rinha-backend-api-01
    build:
      context: ../../
      dockerfile: Dockerfile
    hostname: app1
    environment:
      DB_URL: "postgres://postgres:123456578@postgres:5432/postgres"
      DB_POOL: 35
      REQ_TIMEOUT: 2000
      NODE_ENV: production
      CLUSTER: true
      CLUSTER_WORKERS: 5
      HTTP_PORT: 8080
    expose:
      - "8080"
    depends_on:
      - postgres
    ulimits:
      nproc: 1000000
      nofile:
        soft: 1000000
        hard: 1000000
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: '0.6GB'
  
  app2:
    <<: *app
    container_name: rinha-backend-api-02
    hostname: app2
  
  postgres:
    container_name: rinha-backend-db
    image: postgres:15.4 
    environment:
      POSTGRES_DB: postgres
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: 123456578
    volumes:
      - ./postgresql.conf:/docker-entrypoint-initdb.d/postgresql.conf
    command: postgres -c config_file=/docker-entrypoint-initdb.d/postgresql.conf
    ports:
      - 5432:5432
    healthcheck:
      test: pg_isready -U $$POSTGRES_USER
      interval: 5s
      timeout: 5s
      retries: 20
      start_period: 10s
    deploy:
      resources:
        limits:
          cpus: '0.4'
          memory: '1.5GB'
    
  nginx:
    container_name: rinha-backend-nginx
    image: nginx:alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    depends_on:
      - app1
      - app2
    ports:
      - "9999:9999"
    ulimits:
      nproc: 1000000
      nofile:
        soft: 1000000
        hard: 1000000
    deploy:
      resources:
        limits:
          cpus: '0.1'
          memory: '0.3GB'

nginx.conf 配置 #

worker_processes auto; # create CPU number x 1.5 workers, ex: 4 cpus = 6 workers
worker_rlimit_nofile 500000; # n of file descriptors, each db connection is a socket that is a fd

events {
    worker_connections 1024; # how many simultaneous connections each worker can take
}

http {
    access_log off;
    error_log /dev/null emerg;

    upstream api {
        server app1:8080;
        server app2:8080;
        keepalive 500;
    }

    server {
        listen 9999;

        location / {
            proxy_buffering off;
            proxy_set_header Connection "";
            proxy_http_version 1.1;
            proxy_set_header Keep-Alive "";
            proxy_set_header Proxy-Connection "keep-alive";
            proxy_pass http://api;
        }
    }
}

postgres.conf 配置 #

listen_addresses = '*'
max_connections = 350
superuser_reserved_connections = 3
unix_socket_directories = '/var/run/postgresql'
shared_buffers = 512MB
work_mem = 4MB
maintenance_work_mem = 256MB
effective_cache_size = 1GB
wal_buffers = 64MB
checkpoint_timeout = 10min
checkpoint_completion_target = 0.9
random_page_cost = 4.0
effective_io_concurrency = 2
autovacuum = on
log_statement = 'none'
log_duration = off
log_lock_waits = on
log_error_verbosity = terse
log_min_messages = panic
log_min_error_statement = panic

index.js 代码 #

import http from 'node:http';
import cluster from 'node:cluster';
import handler from './handler.js';

const PORT = process.env.HTTP_PORT || 8080;
const TIMEOUT = Number(process.env.REQ_TIMEOUT) || 5000;
const numForks = Number(process.env.CLUSTER_WORKERS) || 5;

if (cluster.isPrimary && process.env.CLUSTER === 'true') {
    console.log(`index.js: Primary ${process.pid} is running`);
    for (let i = 0; i < numForks; i++)
        cluster.fork();
    cluster.on('exit', (worker, code, signal) => {
        console.log(`index.js: worker ${worker.process.pid} died: code ${code} signal ${signal}`);
    })
} else {
    const server = http.createServer(handler)
        .listen(PORT, () => console.log(`index.js:${process.pid} is running at ${PORT}`));
    
    if (process.env.USE_TIMEOUT === 'true') {
        server.setTimeout(TIMEOUT)
        console.log(`Starting with timeout as ${TIMEOUT}ms`)

        server.on('timeout', (socket) => {
            console.warn(`Timing out connection`);
            socket.end();
        })
    }
}

handler.js 代码 #

import { parse } from 'node:url';
import { DEFAULT_HEADERS } from './utils/server_utils.js';
import { routes } from './routes/personRoute.js';
import PG from './providers/postgres/db/db.js';
import { generateInstance } from './factories/personFactory.js'

const db = new PG();
const personService = generateInstance({ db });

const personRoutes = routes({ personService });

const allRoutes = {
    ...personRoutes,
    default: (req, res) => {
        res.writeHead(404, DEFAULT_HEADERS);
        res.end();
    }
}

async function handler(req, res) {
    const { url, method } = req;
    let { pathname } = parse(url, true);
    pathname = pathname.split('/')[1];
    const route = allRoutes[pathname] ? allRoutes[pathname][method] : allRoutes.default;
    
    await route(req, res);
}

export function closeDB() {
    db.pool.end();
}

export default handler;

personRoute.js 路由代码 #

const routes = ({ personService }) => ({
    person: {
        POST: async (req, res) => {
            try {
                const data = await once(req, 'data');
                const item = JSON.parse(data);
                validatePersonPost(item);
                const person = new Person(item);
                const id = person.id;
        
                await personService.create(person);
                const headers = {
                    ...DEFAULT_HEADERS,
                    'Location': `/person/${id}`
                };
                res.writeHead(201, headers);
                return res.end();
            } catch (err) {
                if (err.code === PG_DUPLICATE_ENTRY_ERR_CODE) {
                    res.writeHead(422, DEFAULT_HEADERS);
                    return res.end();
                }
                if (err instanceof CreatePersonValidationError) {
                    res.writeHead(400, DEFAULT_HEADERS);
                    res.write(err.message);
                    return res.end();
                }
                res.writeHead(500, DEFAULT_HEADERS);
                return res.end();
            }
        },
        GET: async (req, res) => {
            const { url } = req;
            const { pathname, query } = parse(url, true);
            const params = pathname.split('/');

            if (params.length > 2) {
                try {
                    const id = params[2];
                    validatePersonByIdGET(id, res);
                    const person = await personService.findById(id);
                    if (!person)
                        res.writeHead(404, DEFAULT_HEADERS);
                    else
                        res.write(JSON.stringify(person));
                    return res.end();
                } catch (err) {
                    if (err instanceof GetPersonValidationError) {
                        res.writeHead(400, DEFAULT_HEADERS);
                        res.write(err.message);
                        return res.end();
                    }
                    res.writeHead(500, DEFAULT_HEADERS);
                    return res.end();
                }
            } else {
                try {
                    const term = query['t'];
                    if (term) {
                        const people = await personService.findByTerm(term);
                        res.write(JSON.stringify(people));
                        return res.end();
                    } 
                } catch (err) {
                    res.writeHead(500, DEFAULT_HEADERS);
                    return res.end();
                }
            }
            res.writeHead(400, DEFAULT_HEADERS);
            return res.end()
        }
    },
    'person-count': {
        GET: async (req, res) => {
            try {
                const count = await personService.count();
                res.write(JSON.stringify(count));
                return res.end();
            } catch (err) {
                res.writeHead(500, DEFAULT_HEADERS);
                return res.end();
            }
        }
    }
});

export { routes };

内容的提问来源于stack exchange，提问作者Vinicius Bass