eBPF中map ID持续增长的合理性及上限问题咨询

我正在运行eBPF sockops程序，测试期间需要多次加载和重新加载该程序。

用户态加载程序 #

static const char *__doc__ = "User space program to load the sockops bpf program to register sockets
";

#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <unistd.h>

const char *cgroup_dir = "/sys/fs/cgroup/unified";

int main(int argc, char **argv) {
  int err, len;
  int cgroup_fd;
  const char *sockops_file = "bpf_sockops.o";
  struct bpf_object *sockhash_obj;
  bool do_unload = false;

  // Open the cgroup fd -- needed for both attach and detach operations.
  fprintf(stdout, "Opening cgroup file %s\n", cgroup_filename);
  cgroup_fd = open(cgroup_dir, O_RDONLY);
  if (cgroup_fd < 0) {
    fprintf(stderr, "ERR: opening cgroup file %s\n", strerror(errno));
    goto exit_cgroup;
  }

  // Check if the program is to be unloaded.
  if (do_unload) {
    // Unload the sockops program
    err = bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS);
    if (err) {
      goto fail;
    }

    return 0;
  }

  // Open, load and attach sockops_obj if not already attached
  sockhash_obj = bpf_object__open_file(sockops_file, NULL);
  if (libbpf_get_error(sockhash_obj)) {
    goto fail;
  }

  struct bpf_program *sockops_prog = bpf_object__find_program_by_name(sockhash_obj, "bpf_add_to_sockhash");
  if (!sockops_prog) {
    goto fail;
  }

  // Load the sockops program
  err = bpf_object__load(sockhash_obj);
  if (err) {
    goto fail;
  }

  // Attach the sockops program
  // Using core BPF API as libbpf doesn't support sockops yet.
  err = bpf_prog_attach(bpf_program__fd(sockops_prog), cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);
  if (err) {
    fprintf(stderr, "ERR: attaching program\n");
    goto fail;
  }

  fprintf(stdout, "Successfully loaded BPF program.\n");

  return 0;

exit_cgroup:
  close(cgroup_fd);

fail:
  return -1;
}

关键附加/分离调用 #

err = bpf_prog_attach(bpf_program__fd(sockops_prog), cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);
...
err = bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS);

sockops BPF程序 #

// File Name: bpf_sockops.c
#include <linux/in.h>
#include <linux/tcp.h>

#include <linux/bpf.h>
#include <sys/socket.h>

#include <bpf/bpf_endian.h>
#include <bpf/bpf_helpers.h>

char LICENSE[] SEC("license") = "GPL";

// sock_ops_map maps the sock_ops key to a socket descriptor
struct {
  __uint(type, BPF_MAP_TYPE_SOCKHASH);
  __uint(max_entries, 65535);
  __type(key, struct sock_key);
  __type(value, __u64);
} sock_ops_map SEC(".maps");

// `sock_key' is a key for the sockmap
struct sock_key {
  __u32 sip4;
  __u32 dip4;
  __u32 sport;
  __u32 dport;
} __attribute__((packed));

// `sk_extract_key' extracts the key from the `bpf_sock_ops' struct
static inline void sk_extract_key(struct bpf_sock_ops *ops,
                                  struct sock_key *key) {
  key->dip4 = ops->remote_ip4;
  key->sip4 = ops->local_ip4;
  key->sport = (bpf_htonl(ops->local_port) >> 16);
  key->dport = ops->remote_port >> 16;
}

SEC("sockops")
int bpf_add_to_sockhash(struct bpf_sock_ops *skops) {
  __u32 family, op;

  family = skops->family;
  op = skops->op;
  bpf_printk("Got new operation %d for socket.\n", op);

  switch (op) {
    case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
    case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
      if (family == AF_INET) {
        struct sock_key key = {};
        sk_extract_key(skops, &key);
        int ret = bpf_sock_hash_update(skops, &sock_ops_map, &key, BPF_NOEXIST);
        if (ret != 0) {
          bpf_printk("Failed to update sockmap: %d\n", ret);
        } else {
          bpf_printk("Added new socket to sockmap\n");
        }
      }
      break;
    default:
      break;
  }
  return 0;
}

多次重载后的bpftool输出 #

~/ebpf-code$ sudo bpftool prog show
1751: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1752: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1753: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1754: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1755: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1756: cgroup_skb  tag 6deef7357e7b4530  gpl
        loaded_at 2023-06-21T23:42:28-0600  uid 0
        xlated 64B  jited 54B  memlock 4096B
1762: sock_ops  name bpf_add_to_sock  tag 79fcc2296545280f  gpl
        loaded_at 2023-06-21T23:53:36-0600  uid 0
        xlated 880B  jited 521B  memlock 4096B  map_ids 1219,1217,1220
        btf_id 1554

如上述所示，map id已达1217-1220。请问该现象是否符合预期？当达到map id上限时会发生什么？

回答 #

现象是否符合预期？ #

不符合预期。每次重载都生成新的map ID属于BPF资源泄漏问题，旧的程序和映射未被内核正确回收。

核心原因：

• 卸载逻辑未生效：代码中do_unload被硬编码为false，bpf_prog_detach从未执行，旧程序一直挂载在cgroup上。
• 资源未释放：用户态程序未在退出或卸载时调用bpf_object__close释放BPF对象，内核因持有引用无法回收旧映射和程序实例。
• 未清理旧实例：加载新程序前未检查并分离已存在的sockops程序，导致多实例共存，每个实例对应独立映射。

达到map ID上限会发生什么？ #

• 先触发内存限制：实际场景中，会先碰到RLIMIT_MEMLOCK锁定内存上限，加载新程序返回ENOMEM错误，无法创建新映射。
• ID耗尽直接失败：BPF映射ID是32位无符号整数，理论上限为2^32-1。当ID耗尽时，内核无法分配新ID，加载操作直接失败，返回EINVAL或ENOSPC类错误。

修复方案 #

1. 修复卸载逻辑：通过命令行参数（如-u）控制do_unload，确保卸载时执行bpf_prog_detach并调用bpf_object__close释放资源。
2. 加载前清理旧程序：加载新程序前，先尝试分离cgroup上已存在的sockops程序，避免重复挂载。
3. 完善错误路径资源释放：在所有失败退出路径中添加bpf_object__close调用，确保资源被正确释放。

示例修正后的卸载逻辑片段：

// 解析命令行参数设置do_unload
for (int i = 1; i < argc; i++) {
    if (!strcmp(argv[i], "-u")) {
        do_unload = true;
        break;
    }
}

if (do_unload) {
    err = bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS);
    if (err) {
        fprintf(stderr, "ERR: detaching program\n");
    }
    close(cgroup_fd);
    return err ? -1 : 0;
}

// ... 加载逻辑 ...

// 失败路径释放资源
fail:
    if (sockhash_obj) {
        bpf_object__close(sockhash_obj);
    }
    close(cgroup_fd);
    return -1;

内容的提问来源于stack exchange，提问作者diviquery