EPOLLET模式下epoll事件未完全处理的问题排查

我编写了一个简单的客户端-服务器应用，为监听socket设置EPOLLET边缘触发标志后，发现并非所有事件都能被正确处理。测试过程中，客户端循环建立连接并发送数据，共发起10000次连接，但服务器统计的监听socket和客户端socket事件总数始终少于预期（约9200次）。不清楚这一现象是事件处理方式错误导致，还是测试统计方法（通过每次socket事件触发时打印日志计数）存在问题。

客户端代码（client.c） #

#include <assert.h>
#include <netdb.h>
#include <stdio.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>

#define DEF_IP             "127.0.0.1"
#define DEF_PORT           "3940"
#define DEF_MESSAGES_COUNT 10000

typedef struct addrinfo addrinfo;

static int conn_init();

int main() {
    int      i, fd_socket;
    ssize_t  bytes_send;
    uint64_t num;

    for (i = 0; i < DEF_MESSAGES_COUNT; i++) {
        fd_socket = conn_init();

        num = htobe64(i);
        bytes_send = send(fd_socket, &num, sizeof(num), 0);
        assert(bytes_send > -1);

        printf("I! Client: sent [%d], bytes: [%ld]\n", i, bytes_send);
        close(fd_socket);
    }

    printf("sended: %d messages ([0] - [%d])\n", i, i - 1);
    return 0;
}

static int conn_init() {
    addrinfo  info_hints = {0};
    addrinfo *info_server;
    int       fd_socket, ret;

    info_hints.ai_family = AF_UNSPEC;
    info_hints.ai_socktype = SOCK_STREAM;

    ret = getaddrinfo(DEF_IP, DEF_PORT, &info_hints, &info_server);
    assert(ret == 0);

    fd_socket = socket(info_server->ai_family, info_server->ai_socktype, info_server->ai_protocol);
    assert(fd_socket > -1);

    ret = connect(fd_socket, info_server->ai_addr, info_server->ai_addrlen);
    assert(ret == 0);

    freeaddrinfo(info_server);
    return fd_socket;
}

服务器代码（server.c） #

#include <assert.h>
#include <fcntl.h>
#include <netdb.h>
#include <stdbool.h>
#include <stdio.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>

typedef struct addrinfo         addrinfo;
typedef struct epoll_event      epoll_event;
typedef struct sockaddr_storage sockaddr_storage;
typedef struct sockaddr         sockaddr;

#define DEF_IP         "127.0.0.1"
#define DEF_PORT       "3940"
#define DEF_MAX_EVENTS 1000
#define DEF_BACKLOG    1000

static int  conn_handle_socket(int fd_socket);
static void epoll_add(int fd_epoll, int fd, uint32_t flag);

int main() {
    int         fd_socket, fd_connect, fd_epoll, ret;
    epoll_event events[DEF_MAX_EVENTS];
    addrinfo    info_hints = {0};
    addrinfo *  info_server;

    info_hints.ai_family = AF_UNSPEC;      // IPv4 或 IPv6
    info_hints.ai_socktype = SOCK_STREAM;  // TCP
    info_hints.ai_flags = AI_PASSIVE;      // 使用本机IP

    ret = getaddrinfo(DEF_IP, DEF_PORT, &info_hints, &info_server);
    assert(ret == 0);

    fd_socket = socket(info_server->ai_family, info_server->ai_socktype, info_server->ai_protocol);
    assert(fd_socket > -1);

    ret = setsockopt(fd_socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&(int){1}, sizeof(int));
    assert(ret == 0);

    ret = bind(fd_socket, info_server->ai_addr, info_server->ai_addrlen);
    assert(ret == 0);

    freeaddrinfo(info_server);

    ret = listen(fd_socket, DEF_BACKLOG);
    assert(ret == 0);

    // 初始化epoll
    fd_epoll = epoll_create1(0);
    assert(fd_epoll > 0);

    // 监听socket文件描述符
    epoll_add(fd_epoll, fd_socket, EPOLLET);

    printf("I! Server: is ready\n");

    while (1) {
        int num_events = epoll_wait(fd_epoll, events, DEF_MAX_EVENTS, -1);
        assert(num_events > -1);

        for (int i = 0; i < num_events; i++) {
            int fd_tmp = events[i].data.fd;

            if (fd_tmp == fd_socket) {
                printf("I! Server: SOCKET FD\n");
                fflush(stdout);

                // 接受客户端连接
                fd_connect = conn_handle_socket(fd_socket);
                epoll_add(fd_epoll, fd_connect, EPOLLONESHOT);
            } else if (events[i].events & EPOLLIN) {
                printf("I! Server: CLIENT FD\n");
                fflush(stdout);

                // 处理客户端连接
                assert(epoll_ctl(fd_epoll, EPOLL_CTL_DEL, fd_tmp, NULL) == 0);
                close(fd_tmp);
            }
        }
    }

    assert(epoll_ctl(fd_epoll, EPOLL_CTL_DEL, fd_socket, NULL) == 0);
    assert(epoll_ctl(fd_epoll, EPOLL_CTL_DEL, STDIN_FILENO, NULL) == 0);
    close(fd_socket);
    close(fd_epoll);
    return 0;
}

static int conn_handle_socket(int fd_socket) {
    int              fd_connect;
    sockaddr_storage addr_connected;
    socklen_t        sin_size;

    sin_size = sizeof(addr_connected);
    fd_connect = accept(fd_socket, (sockaddr *)&addr_connected, &sin_size);
    assert(fd_connect > -1);  // 忽略EAGAIN || EWOULDBLOCK

    return fd_connect;
}

static void epoll_add(int fd_epoll, int fd, uint32_t flag) {
    int         flags, ret;
    epoll_event ev = {0};

    ev.events = EPOLLIN;
    if (flag == EPOLLET || flag == EPOLLONESHOT) {
        ev.events |= flag;
    }
    ev.data.fd = fd;

    ret = epoll_ctl(fd_epoll, EPOLL_CTL_ADD, fd, &ev);
    assert(ret == 0);

    flags = fcntl(fd, F_GETFL, 0);
    assert(flags > -1);

    flags |= O_NONBLOCK;

    ret = fcntl(fd, F_SETFL, flags);
    assert(ret > -1);
}

核心问题 #

1. EPOLLET模式下未循环调用accept
   边缘触发模式的特性是：监听socket的EPOLLIN事件仅在连接队列从空转为非空时触发一次。如果连接队列中存在多个待处理连接，仅调用一次accept会导致剩余连接不会触发新的EPOLLIN事件，这些连接会被服务器遗漏，直接导致统计数不足。

2. 错误处理accept的EAGAIN返回值
   你通过epoll_add给监听socket设置了非阻塞模式，当连接队列中没有新连接时，accept会返回EAGAIN错误。但代码中用assert(fd_connect > -1)直接终止程序，服务器意外退出后会丢失后续所有事件。

3. 日志统计的潜在误差
   高并发场景下，stdout的缓冲机制可能导致部分日志未及时刷新输出；另外当TCP连接队列（backlog）满时，新的连接请求会被内核直接丢弃，这也会导致实际处理数少于客户端发起的连接数。

修复步骤 #

1. 循环处理监听socket的所有待处理连接 #

修改服务器中监听socket的事件处理逻辑，循环调用accept直到返回EAGAIN：

if (fd_tmp == fd_socket) {
    printf("I! Server: SOCKET FD\n");
    fflush(stdout);

    sockaddr_storage addr_connected;
    socklen_t sin_size = sizeof(addr_connected);
    // 循环accept直到无新连接
    while (1) {
        int fd_connect = accept(fd_socket, (sockaddr *)&addr_connected, &sin_size);
        if (fd_connect == -1) {
            if (errno == EAGAIN || errno == EWOULDBLOCK) {
                break; // 没有更多待处理连接
            }
            perror("accept failed");
            exit(EXIT_FAILURE);
        }
        epoll_add(fd_epoll, fd_connect, EPOLLONESHOT);
    }
}

2. 正确处理accept的错误返回 #

移除conn_handle_socket中的assert，让上层逻辑处理错误：

static int conn_handle_socket(int fd_socket) {
    int              fd_connect;
    sockaddr_storage addr_connected;
    socklen_t        sin_size;

    sin_size = sizeof(addr_connected);
    return accept(fd_socket, (sockaddr *)&addr_connected, &sin_size);
}

3. 优化事件统计方式（可选） #

改用内存计数器统计事件数，避免日志丢失：

// 在main函数开头添加计数器
int listen_events = 0;
int client_events = 0;

// 处理监听事件时
listen_events++;
// 处理客户端事件时
client_events++;

// 可添加信号处理函数，在退出时打印统计结果
static void sigint_handler(int sig) {
    printf("\nTotal listen events: %d, client events: %d\n", listen_events, client_events);
    exit(EXIT_SUCCESS);
}

// 在main中注册信号处理
signal(SIGINT, sigint_handler);

内容的提问来源于stack exchange，提问作者vsezanatodazheeto