使用eBPF Kprobe读取TCP包payload时iov_iter数据异常求解

问题描述 #

我尝试通过libbpf使用eBPF Kprobe挂载到tcp_sendmsg和tcp_recvmsg函数，解析TCP收发数据包的payload。根据认知，payload存储在第二个参数struct msghdr *msg的msg_iter（struct iov_iter*）指向的iov（struct iovec*）的iov_base中。但读取iovec的iov_length时结果为1，而msg_iter->count却是正确的payload长度；从iov_base读取数据时，数据出现损坏。推测数据需完全刷入iov_iter后才能读取，现咨询如何在Kprobe中正确读取TCP包payload？

复现代码（tcp_sendmsg的eBPF Kprobe） #

#include"vmlinux.h"
#include<bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>

const char fmt_info1[] = "segs: %d, offset: %d ,count:%d\n";
const char fmt_info2[] = "iov_length: %d, read_length: %d, data-readsuccess: %d\n";
const char fmt_notify_data[] = "iter_type: %d, Data: ";
const char fmt_data[] = "%02x ";
const char fmt_end[] = "\n";
unsigned char databuf[256];

SEC("kprobe/tcp_sendmsg")
int probe_tcp_sendmsg(struct pt_regs *ctx){
    struct msghdr *msg = (struct msghdr *)PT_REGS_PARM2(ctx);
    struct iov_iter *iter= &(msg->msg_iter);
    struct iovec *iter_data = (struct iovec *)&(iter->iov);
    int read_success=-1;
    long unsigned int iov_seg_count;//count of iovec's
    unsigned char iov_type;//0 is iter_iovec
    unsigned long iov_count;//total data length in iter
    unsigned long iov_length;//length of current iovec
    unsigned long iov_offset;//iovec's to skip in iter
    unsigned long read_length;//final length in the data buffer

    read_success=bpf_probe_read(&iov_type, sizeof(unsigned char), &(iter->iter_type));
    if(read_success!=0) {
        return 0;
    } 
    read_success=bpf_probe_read(&iov_seg_count, sizeof(long unsigned int), &(iter->nr_segs));
    if(read_success!=0) {
        return 0;
    } 
   read_success=bpf_probe_read(&iov_count, sizeof(unsigned long), &(iter->count));
    if(read_success!=0) {
        return 0;
    } 
    read_success=bpf_probe_read(&iov_offset, sizeof(unsigned long), &(iter->iov_offset));
    if(read_success!=0) {
        return 0;
    } 

    void* startaddress;
    read_success=bpf_probe_read(&startaddress, sizeof(void*), &(iter_data->iov_base));
    if(read_success!=0){
        return 0;
    }
    
    
    read_success=bpf_probe_read(&iov_length, sizeof(size_t), &(iter_data->iov_len));
    if(read_success!=0){
        return 0;
    }
    
    if(iov_length< 256){//change against iov_count
        read_length=iov_length;//change against iov_count
    }else{
        read_length=256;
    } 
    read_success=bpf_probe_read(databuf, read_length, startaddress);
    bpf_trace_printk(fmt_info1, sizeof(fmt_info1),iov_seg_count, iov_offset, iov_count);
    bpf_trace_printk(fmt_info2, sizeof(fmt_info2),iov_length, read_length, read_success);
    bpf_trace_printk(fmt_notify_data, sizeof(fmt_notify_data), iov_type);
    for(unsigned long i=0;i<read_length;++i){
        bpf_trace_printk(fmt_data,sizeof(fmt_data), databuf[i]);
    } 
    bpf_trace_printk(fmt_end, sizeof(fmt_end));

    return 0;
}

char LICENSE[] SEC("license") = "Dual BSD/GPL";

编译与加载步骤 #

• 生成vmlinux.h：bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
• 编译eBPF程序：clang -g -O3 -target bpf -D__TARGET_ARCH_x86 -c bpf_ip_tcp.c -o bpf_ip_tcp.o
• 生成骨架文件：bpftool gen skeleton bpf_ip_tcp.o name bpf_ip_tcp > bpf_ip_tcp.h
• 编译加载器：clang -Wall -Wextra -g3 main.c -o ebpf-example -lbpf

加载器代码 #

#include <sys/resource.h>
#include "bpf_ip_tcp.h"
int main(){
    struct rlimit rlim_new = {
        .rlim_cur   = RLIM_INFINITY,
        .rlim_max   = RLIM_INFINITY,
    };
    if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) {
        exit(1);
    }
    struct bpf_ip_tcp *skel = bpf_ip_tcp__open();
    bpf_ip_tcp__load(skel);
    bpf_ip_tcp__attach(skel); 
    while(1==1){}; 
    return 0;
}

测试结果（执行wget时cat /sys/kernel/tracing/trace_pipe输出） #

...
wget-22674   [006] d...1 46410.754999: bpf_trace_printk: segs: 1, offset: 0 ,count:400
wget-22674   [006] d...1 46410.755059: bpf_trace_printk: iov_length: 1, read_length: 1, data-readsuccess: 0
wget-22674   [006] d...1 46410.755061: bpf_trace_printk: iter_type: 0, Data: 
wget-22674   [006] d...1 46410.755061: bpf_trace_printk: 40
...

期望结果：iov_length与count相等，或存在多个段。

系统环境 #

• Ubuntu 22.04 LTS
• 内核版本5.15.0

解决方案 #

1. 理解问题根源 #

挂载tcp_sendmsg的Kprobe时，内核可能还在处理iov_iter的拆分或初始化过程：

• msg_iter->count是payload的总长度，但iter->iov仅代表当前正在处理的单个iovec片段，因此iov_len只会返回当前片段的长度，而非总长度。
• 内核使用iov_iter迭代机制逐步处理数据时，直接读取iov_base可能拿到未完全准备好的内存，导致数据损坏。

2. 使用bpf_iter_read读取完整数据 #

内核5.14及以上版本支持bpf_iter_read helper，它能自动处理多段iovec的拼接，直接从iov_iter读取完整数据。修改后的代码示例：

#include"vmlinux.h"
#include<bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>

const char fmt_info[] = "total length: %lu, read length: %lu\n";
const char fmt_data[] = "%02x ";
const char fmt_end[] = "\n";
unsigned char databuf[256];

SEC("kprobe/tcp_sendmsg")
int probe_tcp_sendmsg(struct pt_regs *ctx){
    struct msghdr *msg = (struct msghdr *)PT_REGS_PARM2(ctx);
    struct iov_iter *iter = &msg->msg_iter;
    unsigned long total_len = iter->count;
    unsigned long read_len = total_len > 256 ? 256 : total_len;
    long ret;

    // 使用bpf_iter_read读取iov_iter中的完整数据
    ret = bpf_iter_read(iter, databuf, read_len);
    if (ret <= 0) {
        bpf_trace_printk("read failed: %ld\n", ret);
        return 0;
    }

    bpf_trace_printk(fmt_info, sizeof(fmt_info), total_len, ret);
    for (unsigned long i = 0; i < ret; ++i) {
        bpf_trace_printk(fmt_data, sizeof(fmt_data), databuf[i]);
    }
    bpf_trace_printk(fmt_end, sizeof(fmt_end));

    return 0;
}

char LICENSE[] SEC("license") = "Dual BSD/GPL";

注意：bpf_iter_read会修改iov_iter的偏移量，若需保留原迭代器状态，建议先通过bpf_probe_read拷贝一份iov_iter结构（需注意结构大小与内核版本兼容性），再对拷贝的结构进行读取操作。

3. 低内核版本备选方案：挂载Kretprobe #

如果内核版本低于5.14不支持bpf_iter_read，可以改为挂载tcp_sendmsg的Kretprobe。此时内核已处理完数据，iov_iter状态更稳定，但需注意：payload可能已被发送，部分用户空间内存可能已被释放，读取前需确认内存有效性。

内容的提问来源于stack exchange，提问作者Nicolai Schmitt