为何bpf_sk_redirect_map无法重定向UDP SKB？技术排查求助

问题 #

编写UDP消息拆分到不同socket的程序，选择sk_skb/verdict钩子点以访问skb->data，但调用bpf_sk_redirect_map重定向报文时始终返回SK_DROP。

内核版本：Linux wusheng 6.11.7-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.11.7-1 (2024-11-09) x86_64 GNU/Linux

最小复现代码 #

eBPF代码

// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_endian.h>

struct
{
    __uint(type, BPF_MAP_TYPE_SOCKMAP);
    __uint(max_entries, 65535);
    __type(key, __u32);
    __type(value, __u64);
} sock_map SEC(".maps");

SEC("sk_skb/verdict")
int prog_skb_verdict(struct __sk_buff *skb)
{
    __u32 zero = skb->local_port;
    
    bpf_printk("socket port %d => %d", skb->local_port, skb->remote_port);
    bpf_printk("socket addr %d => %d", skb->local_ip4, skb->remote_ip4);

    struct bpf_sock * look = bpf_map_lookup_elem(&sock_map, &zero);

    long ret = bpf_sk_redirect_map(skb, &sock_map, zero, BPF_F_INGRESS);
    bpf_printk("socket redir ret=%d sock_find=%d sk_state=%d", ret, look != NULL, look != NULL ? look->state : -1);
    if (look != NULL)
        bpf_sk_release(look);
    return SK_PASS;
}

char _license[] SEC("license") = "GPL";

用户态代码

#include <stdio.h>
#include <stdlib.h>
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <arpa/inet.h>

#define MAP_NAME "/sys/fs/bpf/sock_map"

#define BPF_OBJECT_FILE "test_sockmap_skb_verdict_attach.bpf.o"

int main(int argc, char **argv) {
    struct bpf_object *bpf_obj;
    int prog_fd, sock_map_fd, sock1, sock2;
    struct sockaddr_in addr = {0}, addr2 = {0};

    // 加载 eBPF 对象文件
    bpf_obj = bpf_object__open_file(BPF_OBJECT_FILE, NULL);
    if (!bpf_obj) {
        fprintf(stderr, "Failed to open BPF object file\n");
        return 1;
    }

    if (bpf_object__load(bpf_obj)) {
        fprintf(stderr, "Failed to load BPF program\n");
        return 1;
    }

    // 获取程序的 FD
    prog_fd = bpf_program__fd(bpf_object__find_program_by_name(bpf_obj, "prog_skb_verdict"));
    if (prog_fd < 0) {
        fprintf(stderr, "Failed to find BPF program FD\n");
        return 1;
    }

    // 获取 sock_map 的 FD
    sock_map_fd = bpf_object__find_map_fd_by_name(bpf_obj, "sock_map");
    if (sock_map_fd < 0) {
        fprintf(stderr, "Failed to find sock_map FD\n");
        return 1;
    }
    printf("socket map fd: %d\n", sock_map_fd);

    if (bpf_prog_attach(prog_fd, sock_map_fd, BPF_SK_SKB_VERDICT, 0)) {
        perror("bpf_prog_attach");
        return 1;
    }

    // 创建两个 socket
    sock1 = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
    sock2 = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
    if (sock1 < 0 || sock2 < 0) {
        perror("socket");
        return 1;
    }
    printf("start bind socket\n");

    addr.sin_family = AF_INET;
    addr.sin_port = htons(18080);
    addr.sin_addr.s_addr = htonl(INADDR_ANY);

    addr2.sin_family = AF_INET;
    addr2.sin_port = htons(18081);
    addr2.sin_addr.s_addr = htonl(INADDR_ANY);
    if (bind(sock1, (struct sockaddr *) &addr , sizeof(addr))){
        perror("sock1 bind fail");
        return 1;
    }
    if (bind(sock2, (struct sockaddr *) &addr2 , sizeof(addr2))){
        perror("sock2 bind fail");
        return 1;
    }
    // 将 sockets 添加到 sock_map

#if 1
    uint32_t zero = 18081;
    uint64_t sock1dup = sock1;
    if (bpf_map_update_elem(sock_map_fd, &zero, &sock1dup, BPF_ANY)) {
        perror("bpf_map_update_elem sock1");
        return 1;
    }
    uint32_t zero1 = 18080;
    uint64_t sock2dup = sock2;
    if (bpf_map_update_elem(sock_map_fd, &zero1, &sock2dup, BPF_ANY))
    {
        perror("bpf_map_update_elem sock2");
        return 1;
    }
#endif
    printf("BPF program loaded and sockets added to sock_map.\n");

        //keep listening for data
    int recv_len;
    #define BUFLEN 1500
    char buf[BUFLEN];
    struct sockaddr_in si_other;
    int slen = sizeof(si_other);
    while(1)
    {
        printf("Waiting for data...");
        fflush(stdout);
        
        //try to receive some data, this is a blocking call
        if ((recv_len = recvfrom(sock1, buf, BUFLEN, 0, (struct sockaddr *) &si_other, &slen)) == -1)
        {
            perror("recvfrom()");
        }
        
        //print details of the client/peer and the data received
        printf("Received packet from %x:%d\n", inet_ntoa(si_other.sin_addr), ntohs(si_other.sin_port));
        buf[recv_len] = 0;
        printf("Data: %s\n" , buf);
        
        //now reply the client with the same data
        if (sendto(sock1, buf, recv_len, 0, (struct sockaddr*) &si_other, slen) == -1)
        {
            perror("sendto()");
        }
    }

//    close(sock1);
//    close(sock2);
//    bpf_object__close(bpf_obj);

    return 0;
}

测试日志 #

向18081端口发送UDP报文时的内核日志：

<idle>-0       [005] ..s21   328.378048: bpf_trace_printk: socket port 18081 => 0
          <idle>-0       [005] ..s21   328.378052: bpf_trace_printk: socket addr 0 => 0
          <idle>-0       [005] ..s21   328.378053: bpf_trace_printk: socket redir ret=0 sock_find=1 sk_state=7

内核代码分析 #

查看net/core/sock_map.c中相关逻辑：

static bool sock_map_redirect_allowed(const struct sock *sk)
{
        if (sk_is_tcp(sk))
                return sk->sk_state != TCP_LISTEN;
        else
                return sk->sk_state == TCP_ESTABLISHED;
}

BPF_CALL_4(bpf_sk_redirect_map, struct sk_buff *, skb,
           struct bpf_map *, map, u32, key, u64, flags)
{
        struct sock *sk;
        
        if (unlikely(flags & ~(BPF_F_INGRESS)))
                return SK_DROP;
  
        sk = __sock_map_lookup_elem(map, key);
        if (unlikely(!sk || !sock_map_redirect_allowed(sk)))
                return SK_DROP;
        
        skb_bpf_set_redir(skb, sk, flags & BPF_F_INGRESS);
        return SK_PASS;
}

疑问：不确定内核中sk->sk_state与eBPF中bpf_sock->state是否含义一致，请求排查重定向失败原因。

原因分析与解决方案 #

1. 核心问题：SOCK_MAP对UDP socket的限制 #

从内核代码sock_map_redirect_allowed可以看到，非TCP socket（如UDP）的重定向要求sk->sk_state == TCP_ESTABLISHED，但UDP是无连接协议，其socket状态永远不会达到该值（TCP_ESTABLISHED是TCP专属状态）。

日志中sk_state=7对应UDP绑定后的SS_CONNECTED状态，与内核要求的TCP_ESTABLISHED（数值1）不匹配，导致sock_map_redirect_allowed返回false，最终bpf_sk_redirect_map返回SK_DROP（日志中ret=0对应SK_DROP，内核定义里SK_DROP=0、SK_PASS=1）。

这里明确：bpf_sock->state与内核sk->sk_state完全对应，但UDP和TCP的状态枚举体系不同，内核的判断逻辑本质上禁止了SOCK_MAP对UDP的重定向。

2. 解决方案 #

方案一：改用BPF_MAP_TYPE_SOCKHASH

SOCKHASH支持UDP socket重定向，只要socket处于绑定状态即可。调整点：

• 将eBPF中的map类型改为BPF_MAP_TYPE_SOCKHASH，key需包含IP、端口等标识信息
• 使用bpf_sk_redirect_hash替代bpf_sk_redirect_map执行重定向

方案二：切换钩子点并手动分发

若必须使用SOCK_MAP，可切换到xdp或tc钩子点，绕过状态检查：

• 在钩子点解析UDP报文，手动查找目标socket
• 使用bpf_redirect_sock直接重定向（该函数对UDP无状态限制）

3. 补充说明 #

UDP socket的状态枚举参考内核include/net/tcp_states.h中的enum sock_state，绑定后的UDP状态为SS_CONNECTED（数值7），与TCP状态无交集。

内容的提问来源于stack exchange，提问作者wusheng