Nginx负载下无响应：从HAProxy切换后反向代理连接超时问题求助

大家好，我最近把反向代理从HAProxy换成Nginx后碰到了个头疼的问题：当连接数涨到某个数值时，服务器就彻底不接受新请求了，新连接直接因超时被断开。但离谱的是，换个外部IP来访问就完全正常，而且后端服务本身没有任何问题。

先给大家看看Nginx的debug日志片段：

2023/12/08 16:13:50 [debug] 96834#96834: accept on 0.0.0.0:80, ready: 0

2023/12/08 16:13:50 [debug] 96861#96861: accept() not ready (11: Resource temporarily unavailable)

2023/12/08 16:13:50 [debug] 96860#96860: accept() not ready (11: Resource temporarily unavailable)

2023/12/08 16:13:50 [debug] 96858#96858: accept() not ready (11: Resource temporarily unavailable)

2023/12/08 16:13:50 [debug] 96847#96847: accept() not ready (11: Resource temporarily unavailable)

我顺着日志查了一圈，找到一篇文章说要调高内核限制，但我这台服务器的内核参数已经提前调过了，具体参数如下：

net.core.bpf_jit_enable = 1
net.core.bpf_jit_harden = 1
net.core.bpf_jit_kallsyms = 0
net.core.busy_poll = 0
net.core.busy_read = 0
net.core.default_qdisc = pfifo_fast
net.core.dev_weight = 64
net.core.dev_weight_rx_bias = 1
net.core.dev_weight_tx_bias = 1
net.core.message_burst = 10
net.core.message_cost = 5
net.core.netdev_budget = 600
net.core.netdev_max_backlog = 500000
net.core.netdev_rss_key = 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
net.core.netdev_tstamp_prequeue = 1
net.core.optmem_max = 40960
net.core.rmem_default = 8388608
net.core.rmem_max = 33554432
net.core.rps_sock_flow_entries = 0
net.core.somaxconn = 65535
net.core.warnings = 1
net.core.wmem_default = 4194394
net.core.wmem_max = 33554432
net.core.xfrm_acq_expires = 30
net.core.xfrm_aevent_etime = 10
net.core.xfrm_aevent_rseqth = 2
net.core.xfrm_larval_drop = 1

最后附上我的完整Nginx配置，方便大家帮忙排查问题：

worker_processes  auto;

timer_resolution 100ms;

worker_priority -15;

worker_rlimit_nofile 200000;

error_log  /opt/nginx/error.log debug;

events {
    worker_connections  100000;
    use epoll;
    multi_accept on;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile            on;
    tcp_nodelay         off;
    tcp_nopush          on;

    keepalive_timeout 30 15;
    reset_timedout_connection on;
    send_timeout 120;

    server_tokens off;

    server_names_hash_bucket_size 256;
    server_names_hash_max_size 1024;

    proxy_next_upstream error invalid_header timeout http_500 http_502 http_504;

    gzip on;
    gzip_http_version 1.1;
    gzip_comp_level 5;
    gzip_min_length 1100;
    gzip_buffers 256 32k;
    gzip_types
        # text/html is always compressed by HttpGzipModule
        text/css
        text/javascript
        text/xml
        text/plain
        text/x-component
        application/javascript
        application/x-javascript
        application/json
        application/xml
        application/rss+xml
        application/atom+xml
        font/truetype
        font/opentype
        application/vnd.ms-fontobject
        image/svg+xml;
    gzip_static on;
    gzip_proxied any;
    gzip_proxied            expired no-cache no-store private auth;
    gzip_disable "msie6";
    gzip_vary on;

    proxy_cache_path /tmp/ram/ levels=1:2 use_temp_path=off keys_zone=level-1:20m max_size=26000m inactive=1440m;
    proxy_temp_path /etc/nginx/proxy_temp;
    proxy_cache_key $server_name$request_uri;

    proxy_hide_header X-Powered-By;

    client_body_buffer_size 128k;
    large_client_header_buffers 8 128k;

    proxy_redirect   off;
    proxy_set_header Range "";
    proxy_set_header Connection "";

    proxy_connect_timeout 15;
    proxy_send_timeout    40;
    proxy_read_timeout    66;

    proxy_headers_hash_max_size 1024;
    proxy_headers_hash_bucket_size 128;

    proxy_http_version 1.1;

    proxy_buffer_size   128k;
    proxy_buffers   8 256k;
    proxy_busy_buffers_size   256k;
    proxy_max_temp_file_size 0;

    open_file_cache max=200000 inactive=20s;
    open_file_cache_valid 30s;
    open_file_cache_min_uses 2;
    open_file_cache_errors on;
}

备注：内容来源于stack exchange，提问作者horhe