使用XDP将HTTP GET请求转发至指定端口的问题排查

问题背景 #

尝试通过XDP将发往8000端口的HTTP GET请求转发到289端口的Web服务器，直接访问289端口服务正常，但经XDP转发后无法加载页面。初始阶段TCPDump抓包可见大量RST、PSH包，XDP日志无289端口流量；添加TCP校验和计算后，抓包能看到正常SYN/SYN+ACK交互，但问题仍未解决。

初始代码 #

int xdp_program(struct xdp_md *ctx)
{
    void *data_end = (void *)(long)ctx->data_end;
    void *data = (void *)(long)ctx->data;

    struct ethhdr *eth = data;
    if (eth + 1 > (struct ethhdr *)data_end)
    {
        bpf_printk("Invalid ETHERNET header");
        return XDP_DROP;
    }

    struct iphdr *iph = (data + sizeof(struct ethhdr));
    if (iph + 1 > (struct iphdr *)data_end)
    {
        bpf_printk("Invalid IP header");
        return XDP_DROP;
    }

    if(iph->protocol == IPPROTO_TCP) {
        struct tcphdr *tcph = (data + sizeof(struct ethhdr) + sizeof(struct iphdr));
        if (tcph + 1 > (struct tcphdr *)data_end)
        {
            bpf_printk("Invalid TCP header");
            return XDP_DROP;
        }

        if (tcph->dest == htons(8000))
        {
            if(GetPayload(ctx, eth, iph, tcph) == 1) {
                 tcp->dest = htons(289);
            }    
            return XDP_PASS;
        }
    }
}

更新后的代码（添加TCP校验和） #

static __u16 csum_fold_helper(__u32 csum) {
    csum = (csum & 0xffff) + (csum >> 16);
        return ~((csum & 0xffff) + (csum >> 16));
}

static void update_iph_checksum(struct iphdr *iph) 
{
    uint16_t *next_iph_u16 = (uint16_t *)iph;
    uint32_t csum = 0;
    iph->check = 0;
    #pragma clang loop unroll(full)
    for (uint32_t i = 0; i < sizeof(*iph) >> 1; i++) {
        csum += *next_iph_u16++;
    }

    iph->check = ~((csum & 0xffff) + (csum >> 16));
}
    
int xdp_program(struct xdp_md *ctx)
{
    void *data_end = (void *)(long)ctx->data_end;
    void *data = (void *)(long)ctx->data;

    struct ethhdr *eth = data;
    if (eth + 1 > (struct ethhdr *)data_end)
    {
        bpf_printk("Invalid ETHERNET header");
        return XDP_DROP;
    }

    struct iphdr *iph = (data + sizeof(struct ethhdr));
    if (iph + 1 > (struct iphdr *)data_end)
    {
        bpf_printk("Invalid IP header");
        return XDP_DROP;
    }

    if(iph->protocol == IPPROTO_TCP) {
        struct tcphdr *tcph = (data + sizeof(struct ethhdr) + sizeof(struct iphdr));
        if (tcph + 1 > (struct tcphdr *)data_end)
        {
            bpf_printk("Invalid TCP header");
            return XDP_DROP;
        }

        if (tcph->dest == htons(8000))
        {
            if(GetPayload(ctx, eth, iph, tcph) == 1) {
                // Recalculate checksum.
                __u32 csum;
                __u32 new_dest = ntohs(289);
                csum = bpf_csum_diff(&tcph->dest, sizeof(__u32),
                                &new_dest, sizeof(new_dest), ~tcph->check);
            
                tcph->dest = new_dest;
                tcph->check = csum_fold_helper(csum);
                tcp = tcph;
            }    
            return XDP_PASS;
        }
    }
}

关键遗漏点与解决方案 #

1. 未处理双向流量 #

仅修改请求包的目标端口是不够的，289端口服务器返回的响应包源端口是289，客户端无法将其与自己发往8000端口的请求匹配，导致会话无法完成。必须在XDP中反向处理响应包：将源端口从289改回8000。

2. 未更新IP校验和 #

TCP端口修改后，IP伪头部（用于TCP校验和计算）的内容发生了变化，同时IP自身的校验和也需要重新计算。更新后的代码中虽然有update_iph_checksum函数，但未实际调用，导致IP包校验失败被丢弃。

3. 固定偏移解析头部，未考虑可选字段 #

IP头部长度由iph->ihl字段决定（单位为4字节），而非固定的sizeof(struct iphdr)；TCP头部长度由tcph->doff字段决定（单位为4字节）。固定偏移会导致IP/TCP头部包含可选字段时解析错误，进而引发后续处理失败。

4. Payload解析逻辑可能存在问题 #

GetPayload函数需确保正确识别HTTP GET请求，若逻辑错误会导致该修改的包未被修改，或不该修改的包被错误修改。需确保函数能检查到完整的HTTP起始行（如GET 字段），且包数据长度足够覆盖该字段。

修正后的完整代码示例 #

static __u16 csum_fold_helper(__u32 csum) {
    csum = (csum & 0xffff) + (csum >> 16);
    return ~((csum & 0xffff) + (csum >> 16));
}

static void update_iph_checksum(struct iphdr *iph) 
{
    uint16_t *next_iph_u16 = (uint16_t *)iph;
    uint32_t csum = 0;
    iph->check = 0;
    #pragma clang loop unroll(full)
    // iph->ihl是4字节单位，转成16位字段数为ihl*2
    for (uint32_t i = 0; i < (iph->ihl * 2); i++) {
        csum += *next_iph_u16++;
    }
    iph->check = csum_fold_helper(csum);
}

// 验证是否为HTTP GET请求
static int is_http_get(struct xdp_md *ctx, struct tcphdr *tcph) {
    void *data_end = (void *)(long)ctx->data_end;
    char *payload = (char *)tcph + tcph->doff * 4;
    // 确保Payload有足够长度包含"GET "
    if (payload + 4 > data_end) {
        return 0;
    }
    return (memcmp(payload, "GET ", 4) == 0);
}

int xdp_program(struct xdp_md *ctx)
{
    void *data_end = (void *)(long)ctx->data_end;
    void *data = (void *)(long)ctx->data;

    struct ethhdr *eth = data;
    if ((void *)eth + sizeof(*eth) > data_end) {
        bpf_printk("Invalid Ethernet header");
        return XDP_DROP;
    }

    struct iphdr *iph = (struct iphdr *)(data + sizeof(*eth));
    // 检查IP头部完整（包含可选字段）
    if ((void *)iph + iph->ihl * 4 > data_end) {
        bpf_printk("Invalid IP header");
        return XDP_DROP;
    }

    if (iph->protocol != IPPROTO_TCP) {
        return XDP_PASS;
    }

    struct tcphdr *tcph = (struct tcphdr *)((void *)iph + iph->ihl * 4);
    // 检查TCP头部完整（包含可选字段）
    if ((void *)tcph + tcph->doff * 4 > data_end) {
        bpf_printk("Invalid TCP header");
        return XDP_DROP;
    }

    // 处理请求：目标8000端口的GET请求，转发到289
    if (tcph->dest == htons(8000)) {
        if (is_http_get(ctx, tcph)) {
            __be16 old_port = tcph->dest;
            __be16 new_port = htons(289);
            // 更新TCP校验和
            __u32 csum = bpf_csum_diff(&old_port, sizeof(old_port), 
                                      &new_port, sizeof(new_port), ~tcph->check);
            tcph->dest = new_port;
            tcph->check = csum_fold_helper(csum);
            // 更新IP校验和
            update_iph_checksum(iph);
        }
        return XDP_PASS;
    }

    // 处理响应：源289端口的包，改回源端口8000
    if (tcph->source == htons(289)) {
        __be16 old_port = tcph->source;
        __be16 new_port = htons(8000);
        __u32 csum = bpf_csum_diff(&old_port, sizeof(old_port), 
                                  &new_port, sizeof(new_port), ~tcph->check);
        tcph->source = new_port;
        tcph->check = csum_fold_helper(csum);
        update_iph_checksum(iph);
        return XDP_PASS;
    }

    return XDP_PASS;
}

验证步骤 #

1. 加载修正后的XDP程序到目标网卡
2. 客户端访问8000端口，同时用TCPDump分别抓取客户端侧、服务器侧（289端口）的流量
3. 检查请求包目标端口是否被修改为289，响应包源端口是否被改回8000
4. 验证客户端能否正常加载页面

内容的提问来源于stack exchange，提问作者Fabian