eBPF统计L1d缓存计数与perf stat结果严重不符的问题排查请求

大家好，我最近写了一段eBPF代码，用来统计目标进程的缓存访问与缺失次数，但统计结果和perf stat的输出完全对不上——甚至缓存访问和缺失的数值都非常接近，这明显不符合预期。我需要大家帮忙定位问题出在哪里。

先说明下我的运行环境：
我使用的是KVM虚拟机，内核版本为6.8.0-78-generic，底层宿主机是Intel Xeon Silver 4208 CPU，具体硬件信息如下：

CPU op-mode(s): 32-bit, 64-bit
Address sizes: 46 bits physical, 48 bits virtual
Byte Order: Little Endian
CPU(s): 32
On-line CPU(s) list: 0-31
Vendor ID: GenuineIntel
Model name: Intel(R) Xeon(R) Silver 4208 CPU @ 2.10GHz
CPU family: 6
Model: 85
Virtualization: VT-x
L1d cache: 512 KiB (16 instances)
L1i cache: 512 KiB (16 instances)
L2 cache: 16 MiB (16 instances)
L3 cache: 22 MiB (2 instances)
NUMA node(s): 2

我贴出部分核心代码片段，麻烦大家帮忙检查基础逻辑是否存在错误：

用户态加载器代码（userspace_loader.c） #

#define PREP_CACHE_CFG(type, op, res) \
    ((type) | ((op) << 8) | ((res) << 16))

#define L1D_ACCESS_CFG PREP_CACHE_CFG(PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ, PERF_COUNT_HW_CACHE_RESULT_ACCESS)
#define L1D_MISS_CFG PREP_CACHE_CFG(PERF_COUNT_HW_CACHE_L1D, PERF_COUNT_HW_CACHE_OP_READ, PERF_COUNT_HW_CACHE_RESULT_MISS)
#define L2D_ACCESS_CFG PREP_CACHE_CFG(PERF_COUNT_HW_CACHE_LL, PERF_COUNT_HW_CACHE_OP_READ, PERF_COUNT_HW_CACHE_RESULT_ACCESS)
#define L2D_MISS_CFG PREP_CACHE_CFG(PERF_COUNT_HW_CACHE_LL, PERF_COUNT_HW_CACHE_OP_READ, PERF_COUNT_HW_CACHE_RESULT_MISS)

#define SET_CAMOPT(optdata, name, val) (optdata ? optdata->opts.cam_opts.name : val);

typedef enum {
    INV_CACHE_EVT = -1,
    GEN_REF,    /* Cache Read */
    GEN_MIS,    /* Cache Miss */
    L1D_REF,    /* L1 Refer */
    L1D_MIS,    /* L1 Miss */
    L2D_REF,    /* L2 Refer */
    L2D_MIS,    /* L2 Miss */
    MAX_CACHE_EVTS,
    TOTAL_CACHE_EVTS = MAX_CACHE_EVTS + 1
} cache_evt_t;

int get_cachecfg (cache_evt_t evt) {
    switch(evt) {
        case GEN_REF: return PERF_COUNT_HW_CACHE_REFERENCES;
        case GEN_MIS: return PERF_COUNT_HW_CACHE_MISSES;
        case L1D_REF: return L1D_ACCESS_CFG;
        case L1D_MIS: return L1D_MISS_CFG;
        case L2D_REF: return L2D_ACCESS_CFG;
        case L2D_MIS: return L2D_MISS_CFG;
        default: return -1;
    }
    return 0;
}

int open_dcam_event (int type, int config, pid_t pid, evt_opts_t *opts) {
    struct perf_event_attr attr;
    int group_fd = SET_CAMOPT (opts, group_fd, -1); // 创建事件组
    unsigned long flags = SET_CAMOPT (opts, flags, 0); // 操作标志

    memset(&attr, 0, sizeof (attr));
    attr.type = type;
    attr.size = sizeof (attr);
    attr.config = config;
    attr.sample_period = 1; // 统计每一次事件
    attr.sample_type = PERF_SAMPLE_IDENTIFIER;
    attr.disabled = 1;
    attr.exclude_kernel = 1;
    attr.exclude_hv = 1;
    attr.exclude_guest = 1;
    // attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED|PERF_FORMAT_TOTAL_TIME_RUNNING;

    // 打开perf事件
    int fd = syscall (__NR_perf_event_open, &attr, pid, -1, group_fd, 0x8);
    if (fd < 0) {
        perror ("ERROR: syscall(perf_event_open) failed \n");
        return -1;
    }
    printf ("\nCache Configuration Applied on pid(%d): fd(%d), type(%d) config( 0x%x), under group(%d) \n", pid, fd, type, config, group_fd);
    return fd;
}

int openat_dcam_event (cache_evt_t reg_evt, pid_t pid, void* bpfdata, evt_opts_t *opts) {
    int cfd;
    struct bpf_link *link_stat = NULL;

    cfd = open_dcam_event (PERF_TYPE_HW_CACHE, get_cachecfg (reg_evt), pid, opts);
    if (cfd < 0) return cfd;

    switch(reg_evt) {
        case L1D_REF:
            link_stat = (struct bpf_link *) bpf_program__attach_perf_event (((bpfobj_t*) bpfdata)->progs.on_l1d_access,cfd);
            if (link_stat) ((bpfobj_t*) bpfdata)->links.on_l1d_access = link_stat;
            else return -1;
            break;
        case L1D_MIS:
            link_stat = (struct bpf_link *) bpf_program__attach_perf_event (((bpfobj_t*) bpfdata)->progs.on_l1d_miss,cfd);
            if (link_stat) ((bpfobj_t*) bpfdata)->links.on_l1d_miss = link_stat;
            else return -1;
            break;
    }
}

int main() {
    struct camcs_bpf *bpf_obj;
    bpf_obj = camcs_bpf__open_and_load ();
    if (!bpf_obj) {
        fprintf (stderr, "Failed to open and load BPF skeleton\n");
        return 1;
    }

    fd_l1ac = openat_dcam_event (L1D_REF, pid, bpf_obj, NULL);
    if (fd_l1ac < 0) {
        fprintf (stderr, "Failed to open and attach L1D_REF\n");
        goto cleanup;
    }
    printf ("\nfd_l1ac (%d) for L1D_REF (%d)\n", fd_l1ac, L1D_REF);

    options.opts.cam_opts.group_fd = fd_l1ac;
    fd_l1mc = openat_dcam_event (L1D_MIS, pid, bpf_obj, &options);
    if (fd_l1mc < 0) {
        fprintf (stderr, "Failed to open and attach L1D_MIS\n");
        goto cleanup;
    }
    printf ("\nfd_l1mc (%d) for L1D_MIS (%d)\n", fd_l1mc, L1D_MIS);

    u32 kl1dm = L1D_MIS, kl1da = L1D_REF;
    u64 kl1dm_cntr = 0, kl1da_cntr = 0;

    while (running) {
        sleep (2);
        kl1dm_cntr = 0;
        kl1da_cntr = 0;

        // 从eBPF map中读取计数
        err = bpf_map_lookup_elem (bpf_map__fd(bpf_obj->maps.counts), &kl1dm, &kl1dm_cntr);
        if (err != 0) {
            fprintf (stderr, "Failed to read L1D cache misses: %s\n", strerror(-err));
        }
        err = bpf_map_lookup_elem (bpf_map__fd (bpf_obj->maps.counts), &kl1da, &kl1da_cntr);
        // 省略后续计数输出逻辑
    }
cleanup:
    // 资源清理逻辑
    return 0;
}

我怀疑可能是在perf事件配置、eBPF程序挂载或者计数逻辑上存在基础错误，比如perf_event_open的参数设置、缓存事件配置宏是否正确，或者事件组的使用有没有问题？麻烦大家帮我排查下，谢谢！

内容来源于stack exchange