DPDK计算Mellanox ConnectX-6网卡RSS哈希值不匹配问题求助

云上老周

2026-7-1

复现Mellanox ConnectX-6 Dx网卡RSS哈希值以实现跨网卡负载均衡

我使用Mellanox ConnectX-6 Dx（型号MT2892 Family [ConnectX-6 Dx] 101d，网卡接口ens5f1，驱动mlx5_core）搭配DPDK 22版本，已通过RTE Flow配置端口多队列，基于IP+Port进行流量分流。现在需要复现网卡生成的RSS哈希值，实现跨网卡的流量负载均衡（依据数据包内部的GRE封装后内容而非外层IP/传输层信息），但目前自行计算的哈希值与mbuf中保存的不一致。

已配置的RTE Flow分流代码

/*rte flow*/
const int MAX_PATTERN_IN_FLOW = 10;
const int MAX_ACTIONS_IN_FLOW = 10;

struct rte_flow_attr attr;

struct rte_flow_item pattern[MAX_PATTERN_IN_FLOW];
struct rte_flow_action actions[MAX_ACTIONS_IN_FLOW];
struct rte_flow *flow;
struct rte_flow_error error;

memset(pattern, 0, sizeof(pattern));
memset(actions, 0, sizeof(actions));

/* Set the rule attribute, only ingress packets will be checked. */
memset(&attr, 0, sizeof(struct rte_flow_attr));
attr.ingress = 1;

pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
pattern[0].spec = NULL;

pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
pattern[1].spec = NULL;

pattern[2].type = RTE_FLOW_ITEM_TYPE_GRE;
pattern[2].spec = NULL;

pattern[3].type = RTE_FLOW_ITEM_TYPE_ETH;
pattern[3].spec = NULL;

pattern[4].type = RTE_FLOW_ITEM_TYPE_IPV4;
pattern[4].spec = NULL;

pattern[5].type = RTE_FLOW_ITEM_TYPE_UDP;
pattern[5].spec = NULL;

// end the pattern array 
pattern[6].type = RTE_FLOW_ITEM_TYPE_END;

struct rte_flow_action_rss rss_conf;
uint16_t queues[pi_nNumRxQueues];
rss_conf.func = RTE_ETH_HASH_FUNCTION_DEFAULT;
uint64_t hf = RTE_ETH_RSS_IP | RTE_ETH_RSS_TCP | RTE_ETH_RSS_UDP | RTE_ETH_RSS_SCTP;
hf &= pi_devInfo.flow_type_rss_offloads;
rss_conf.types = hf;
rss_conf.queue_num = pi_nNumRxQueues;
for (int nqQueueIndex= 0; nqQueueIndex < pi_nNumRxQueues; nqQueueIndex++)
        queues[nqQueueIndex] = nqQueueIndex;

rss_conf.queue = queues;

rss_conf.key_len = 0;
rss_conf.key = NULL;
rss_conf.level = 2;

// create the drop action 
actions[0].type = RTE_FLOW_ACTION_TYPE_RSS;
actions[0].conf = &rss_conf;
actions[1].type = RTE_FLOW_ACTION_TYPE_END;

// validate and create the flow rule 
if (rte_flow_validate(pi_nPort, &attr, pattern, actions, &error)==0)
{
    flow = rte_flow_create(pi_nPort, &attr, pattern, actions, &error);
    if(flow){/*success*/}
    else{/*error*/}
}
else {/*error*/}

自行尝试的哈希计算代码

uint8_t rss_hash_default_key[] = {
    0x2c, 0xc6, 0x81, 0xd1,
    0x5b, 0xdb, 0xf4, 0xf7,
    0xfc, 0xa2, 0x83, 0x19,
    0xdb, 0x1a, 0x3e, 0x94,
    0x6b, 0x9e, 0x38, 0xd9,
    0x2c, 0x9c, 0x03, 0xd1,
    0xad, 0x99, 0x44, 0xa7,
    0xd9, 0x56, 0x3d, 0x59,
    0x06, 0x3c, 0x25, 0xf3,
    0xfc, 0x1f, 0xdc, 0x2a,
};

static inline uint32_t
do_softrss(struct rte_mbuf *m)
{
    uint32_t input_len;
    struct rte_ipv4_tuple ipv4_tuple;
    
    char * pRawPacket = static_cast<char*>(rte_pktmbuf_mtod(pi_mbuf, void* ));
    IpHeader * pIpHeader = (IpHeader *)(pRawPacket + offsetOfIp);
    if(pIpHeader->GetVersion()==4)
    {
        ipv4_tuple.src_addr = rte_be_to_cpu_32(pIpHeader->dwSrcAddressBigEndian);
        ipv4_tuple.dst_addr = rte_be_to_cpu_32(pIpHeader->dwDstAddressBigEndian);
        ipv4_tuple.sport = *(uint16_t*)(pRawPacket + transportLayerOffset);
        ipv4_tuple.dport = *(uint16_t*)(pRawPacket + transportLayerOffset+2);
        input_len = RTE_THASH_V4_L3_LEN;
        return rte_softrss_be((uint32_t *)&ipv4_tuple, input_len, rss_key_be);

    }
    return 0;
}
new_rss = do_softrss(mbuf_pointer);
std::cout<< std::hex << mbuf_pointer->hash.rss << " -> " << new_rss << std::dec << std::endl;

哈希值不一致的运行结果

5ed28a5c -> 33eb33eb
974c1896 -> 24e224e2
1edf1638 -> 21752175
8a54c19 -> 80638063
459a6f76 -> 1b351b35
1cdf1d1c -> e53be53b

网卡信息

ethtool -i ens5f0
driver: mlx5_core
version: 5.8-3.0.7
firmware-version: 22.32.2004 (MT_0000000437)
expansion-rom-version:
bus-info: 0000:83:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes

问题分析与解决步骤

核心问题

哈希字段错误：RTE Flow配置的level=2是针对GRE封装后的内层IPv4+UDP做RSS，但自行计算时仅使用了外层IP和端口，完全未处理内层数据包内容。
哈希函数与密钥不匹配：ConnectX-6 Dx默认使用Toeplitz哈希，需确保软件计算的函数、密钥与硬件一致。
字节序处理错误：硬件RSS直接使用网络字节序计算，软件中不应转换为CPU字节序。

修正方案

1. 正确提取内层数据包字段

修改哈希计算代码，解析到GRE后的内层IPv4和UDP信息：

#include <rte_ip.h>
#include <rte_udp.h>

// 简化GRE头结构（仅处理必要字段）
struct gre_hdr {
    uint16_t flags;
    uint16_t protocol;
};

static inline uint32_t
do_softrss(struct rte_mbuf *m)
{
    char *pRawPacket = rte_pktmbuf_mtod(m, char*);
    uint32_t offset = 14; // 外层ETH头长度

    // 解析外层IPv4头
    struct rte_ipv4_hdr *outer_ip = (struct rte_ipv4_hdr*)(pRawPacket + offset);
    if (outer_ip->version != 4) return 0;
    offset += outer_ip->ihl * 4;

    // 解析GRE头，跳过可选字段
    struct gre_hdr *gre = (struct gre_hdr*)(pRawPacket + offset);
    offset += 4;
    if (gre->flags & 0x8000) offset += 2; // 跳过校验和字段
    if (gre->flags & 0x4000) offset += 4; // 跳过密钥字段
    if (gre->flags & 0x2000) offset += 4; // 跳过序列号字段

    offset += 14; // 内层ETH头长度
    // 解析内层IPv4头
    struct rte_ipv4_hdr *inner_ip = (struct rte_ipv4_hdr*)(pRawPacket + offset);
    if (inner_ip->version != 4) return 0;
    offset += inner_ip->ihl * 4;

    // 解析内层UDP头
    struct rte_udp_hdr *inner_udp = (struct rte_udp_hdr*)(pRawPacket + offset);

    // 构造哈希元组（保留网络字节序，不转换为CPU字节序）
    struct rte_ipv4_tuple ipv4_tuple;
    ipv4_tuple.src_addr = inner_ip->src_addr;
    ipv4_tuple.dst_addr = inner_ip->dst_addr;
    ipv4_tuple.sport = inner_udp->src_port;
    ipv4_tuple.dport = inner_udp->dst_port;

    // 使用L3+L4完整长度计算哈希
    uint32_t input_len = RTE_THASH_V4_L3_LEN + RTE_THASH_V4_L4_LEN;
    return rte_softrss_be((uint32_t *)&ipv4_tuple, input_len, rss_hash_default_key);
}

2. 对齐网卡哈希配置

修改RTE Flow中的RSS配置，显式指定Toeplitz哈希函数：

rss_conf.func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; // 替换默认值

若网卡自定义过RSS密钥，通过以下命令获取硬件密钥并替换代码中的rss_hash_default_key：

ethtool -x ens5f1

3. 验证结果

修改后重新运行，计算出的new_rss应与mbuf->hash.rss完全一致。若仍有差异，需检查：

GRE可选字段的解析是否覆盖所有场景
内层数据包的偏移计算是否准确
RSS类型配置（rss_conf.types）是否与软件计算的字段对应

内容的提问来源于stack exchange，提问作者yaron

DPDK计算Mellanox ConnectX-6网卡RSS哈希值不匹配问题求助

已配置的RTE Flow分流代码 #

自行尝试的哈希计算代码 #

哈希值不一致的运行结果 #

网卡信息 #

问题分析与解决步骤 #

核心问题 #

修正方案 #