为何TCP Socket往返时延低于Linux网络栈时延？

我搭建了一套实验环境：两台机器通过以太网直连，均配备10Gbps网卡（IXGBE驱动），且处于同一网络可互相ping通。我用一套用户态-内核态网络栈时延测量代码得到的结果约为100微秒，但自己编写的TCP Socket客户端/服务端在直连时的往返时延（Round Trip Time，RTT）居然低于这个数值，对此我感到困惑。

服务端代码（server.c） #

#define PORT            8080
#define BUFFER_SIZE     1024
#define PACKET_SIZE     1024

int main() {
    int     socket_fd, new_socket, rc;

    // 创建套接字文件描述符
    socket_fd = socket(AF_INET, SOCK_STREAM, 0);
    if (socket_fd  == 0) {
        perror("Socket creation error");
        exit(EXIT_FAILURE);
    }


    // 设置套接字选项
    int opt = 1;
    rc = setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt));
    if (rc < 0) {
        perror("Setsockopt failed");
        exit(EXIT_FAILURE);
    }

    // 绑定地址与端口
    struct sockaddr_in address;
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = INADDR_ANY;
    address.sin_port = htons(PORT);

    rc = bind(socket_fd, (struct sockaddr *)&address, sizeof(address));
    if (rc < 0) {
        perror("Socket bind failed");
        exit(1);
    }

    printf("Listening to %s:%d\n", inet_ntoa(address.sin_addr), ntohs(address.sin_port));
    if (listen(socket_fd, 3)) {
        perror("Listen failed");
        exit(EXIT_FAILURE);
    }

    // 接受连接
    int addrlen = sizeof(address);
    new_socket = accept(socket_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen);
    if (new_socket < 0) {
        perror("Accept failure");
        exit(EXIT_FAILURE);
    }

    // 打印客户端地址
    char ip_address[INET_ADDRSTRLEN];
    inet_ntop(AF_INET, &(address.sin_addr), ip_address, INET_ADDRSTRLEN);
    printf("New connection from %s:%d\n", ip_address, ntohs(address.sin_port));
    
    // 循环接收并回复数据
    int val_read, val_send;
    char    buff[BUFFER_SIZE];
    while (1) {
        memset(buff, 0, sizeof(buff));
        val_read = read(new_socket, buff, sizeof(buff));
        if(val_read == -1) {
            perror("Read failed");
            exit(EXIT_FAILURE);
        }
        else if (val_read > 0) {
            val_send = send(new_socket, buff, sizeof(buff), 0);
            if (val_send == -1) {
                perror("Send failed");
                exit(EXIT_FAILURE);
            }
        }
    }
    close(socket_fd);
    return 0;
}

客户端代码（client.c） #

#define PORT            8080
#define NUM_MESSAGES    1024
#define ROLLING_COUNT   32
#define PACKET_SIZE     32
#define BUFFER_SIZE     1024
#define DEST_ADDRESS    "192.168.10.223"

int main() {
    int sock = 0;
    int total_latency = 0;

    struct sockaddr_in serv_addr;
    struct in_addr dest_addr; 
    memset(&dest_addr, 0, sizeof(struct in_addr));
    inet_pton(AF_INET, DEST_ADDRESS, &dest_addr);


    serv_addr.sin_addr = dest_addr;
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(PORT);


    // 打印目标信息
    printf("Destination address is %s\n", inet_ntoa(dest_addr));
    printf("Destination port    is %d\n", PORT);
    
    // 创建套接字
    sock = socket(AF_INET, SOCK_STREAM, 0);
    if (sock < 0) {
        perror("Socket creation error");
        exit(EXIT_FAILURE);
    }

    // 连接服务端
    int c = connect(sock, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
    if (c < 0) {
        perror("Connection failed");
        exit(EXIT_FAILURE);
    }
    printf("Connected to server!\n");


    // 定义时间变量与缓冲区
    struct timeval start_time, end_time;
    char message[PACKET_SIZE] = {};
    char buffer[BUFFER_SIZE] = {};

    memset(message, 'A', strlen(message));
    double latency;

    // 开始发送并接收数据
    printf("Ready to send/recv messages\n");

    for (int i = 0; i < NUM_MESSAGES; i++) {
        gettimeofday(&start_time, NULL);

        if (send(sock, message, sizeof(message), 0) == -1) {
            perror("Send failed");
            continue;
        }

        if (recv(sock, buffer, sizeof(buffer), SO_RCVTIMEO) == -1) {
            perror("Recv failed");
            continue;
        }

        gettimeofday(&end_time, NULL);
        latency = (end_time.tv_sec - start_time.tv_sec) * 1000000 + 
                  (end_time.tv_usec - start_time.tv_usec);
        printf("%d Message sent at %d.%06d, recv at %d.%06d, latency %06d\n", 
            i, start_time.tv_sec, start_time.tv_usec, end_time.tv_sec, end_time.tv_usec, (int)latency);
    }
    printf("Finished!");
    close(sock);
    return 0;
}

异常现象与疑问 #

我测量的时延是客户端发送消息到收到回复的时间差：

• 使用连接共享网络的eth0接口时，往返时延约900-1000微秒，结果合理；
• 但使用直连以太网的接口时，往返时延低于80微秒，这让我无法理解。

另外还有一个奇怪的现象：启动客户端/服务端后，ping的时延会显著降低，关闭服务端后ping时延恢复正常，这是为什么？

内容的提问来源于stack exchange，提问作者akastack