使用JOLT Transform实现存储控制器与对应温度传感器的JSON转换

需求说明 #

输入JSON包含storage_controller数组和独立的storage_controller_temp数组，每个storage_controller_temp的index字段为"控制器编号.传感器编号"格式字符串（如"1.3"表示属于编号1的存储控制器）。需将对应存储控制器的温度传感器数据嵌入到该控制器对象的temp_sensors数组中。

输入JSON #

{"fragmented": true,"nodeIp": "192.168.0.20","requestId": "35daa014-d995-47ab-885b-0ce5ef24a214","nodeId": "143898594","timestamp": 1717413230106,"requestTimestamp": 1717413185926,"data": {"storage_controller": [{"ctlr_status": "up","ctlr_name": "Top Controller","ctlr_ip_address": "192.168.0.21","ctlr_force_trap": "Empty","ctlr_service_tag": "FSHVH82","ctlr_asset_tag": "","index": "1","ctlr_leader": 1,"ctlr_number": 1,"ctlr_model": "CT_SC4020"},{"ctlr_leader": 2,"ctlr_ip_address": "192.168.0.22","ctlr_model": "CT_SC4020","ctlr_service_tag": "FSHVH82","index": "2","ctlr_number": 2,"ctlr_status": "up","ctlr_name": "Bottom Controller","ctlr_force_trap": "Empty","ctlr_asset_tag": ""}],"storage_controller_temp": [{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 3,"ctlr_temp_status": "up","ctlr_temp_current_c": 38,"ctlr_temp_name": "DIMM2","index": "1.3"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 4,"ctlr_temp_status": "up","ctlr_temp_current_c": 45,"ctlr_temp_name": "PCIe Zone Inlet","index": "2.4"},{"ctlr_temp_status": "up","ctlr_temp_current_c": 40,"ctlr_temp_name": "CPU Outlet","index": "2.6","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 6},{"ctlr_temp_current_c": 35,"ctlr_temp_name": "DIMM2","index": "2.3","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 3,"ctlr_temp_status": "up"},{"ctlr_temp_current_c": 35,"ctlr_temp_name": "DIMM1","index": "2.2","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 2,"ctlr_temp_status": "up"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 2,"ctlr_temp_status": "up","ctlr_temp_current_c": 38,"ctlr_temp_name": "DIMM1","index": "1.2"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 5,"ctlr_temp_status": "up","ctlr_temp_current_c": 45,"ctlr_temp_name": "PCIe Zone Outlet","index": "1.5"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 1,"ctlr_temp_status": "up","ctlr_temp_current_c": 62,"ctlr_temp_name": "CPU One","index": "1.1"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 6,"ctlr_temp_status": "up","ctlr_temp_current_c": 43,"ctlr_temp_name": "CPU Outlet","index": "1.6"},{"ctlr_temp_name": "BBU","index": "1.7","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 7,"ctlr_temp_status": "up","ctlr_temp_current_c": 32},{"ctlr_temp_number": 4,"ctlr_temp_status": "up","ctlr_temp_current_c": 48,"ctlr_temp_name": "PCIe Zone Inlet","index": "1.4","ctlr_temp_force_trap": "Empty"},{"index": "2.5","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 5,"ctlr_temp_status": "up","ctlr_temp_current_c": 41,"ctlr_temp_name": "PCIe Zone Outlet"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 7,"ctlr_temp_status": "up","ctlr_temp_current_c": 30,"ctlr_temp_name": "BBU","index": "2.7"},{"ctlr_temp_name": "CPU One","index": "2.1","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 1,"ctlr_temp_status": "up","ctlr_temp_current_c": 52}]}}

目标输出JSON #

{"fragmented": true,"nodeIp": "192.168.0.20","requestId": "3b93f973-0b88-4464-8208-8ff6dc27189d","nodeId": "143898594","timestamp": 1717078433124,"requestTimestamp": 1717078414005,"data": {"storage_controller": [{"ctlr_asset_tag": "","ctlr_leader": 2,"ctlr_status": "up","ctlr_ip_address": "192.168.0.22","ctlr_force_trap": "Empty","ctlr_service_tag": "FSHVH82","ctlr_number": 2,"ctlr_name": "Bottom Controller","ctlr_model": "CT_SC4020","index": "2","temp_sensors": [{"ctlr_temp_status": "up","ctlr_temp_current_c": 48,"ctlr_temp_name": "CPU One","index": "2.1","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 1},{"index": "2.2","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 2,"ctlr_temp_status": "up","ctlr_temp_current_c": 34,"ctlr_temp_name": "DIMM1"},{"ctlr_temp_status": "up","ctlr_temp_current_c": 34,"ctlr_temp_name": "DIMM2","index": "2.3","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 3},{"ctlr_temp_name": "PCIe Zone Inlet","index": "2.4","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 4,"ctlr_temp_status": "up","ctlr_temp_current_c": 44},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 5,"ctlr_temp_status": "up","ctlr_temp_current_c": 41,"ctlr_temp_name": "PCIe Zone Outlet","index": "2.5"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 6,"ctlr_temp_status": "up","ctlr_temp_current_c": 40,"ctlr_temp_name": "CPU Outlet","index": "2.6"},{"ctlr_temp_status": "up","ctlr_temp_current_c": 29,"ctlr_temp_name": "BBU","index": "2.7","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 7}]},{"ctlr_leader": 1,"ctlr_number": 1,"ctlr_status": "up","ctlr_name": "Top Controller","ctlr_ip_address": "192.168.0.21","ctlr_force_trap": "Empty","ctlr_model": "CT_SC4020","ctlr_service_tag": "FSHVH82","ctlr_asset_tag": "","index": "1","temp_sensors": [{"ctlr_temp_name": "CPU One","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 1,"ctlr_temp_status": "up","ctlr_temp_current_c": 61,"index": "1.1"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 2,"ctlr_temp_status": "up","ctlr_temp_current_c": 38,"ctlr_temp_name": "DIMM1","index": "1.2"},{"ctlr_temp_current_c": 37,"ctlr_temp_name": "DIMM2","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 3,"ctlr_temp_status": "up","index": "1.3"},{"ctlr_temp_status": "up","ctlr_temp_current_c": 48,"ctlr_temp_name": "PCIe Zone Inlet","ctlr_temp_force_trap": "Empty","ctlr_temp_number": 4,"index": "1.4"},{"ctlr_temp_number": 5,"ctlr_temp_status": "up","ctlr_temp_current_c": 45,"ctlr_temp_name": "PCIe Zone Outlet","ctlr_temp_force_trap": "Empty","index": "1.5"},{"ctlr_temp_number": 6,"ctlr_temp_status": "up","ctlr_temp_current_c": 43,"ctlr_temp_name": "CPU Outlet","ctlr_temp_force_trap": "Empty","index": "1.7"},{"ctlr_temp_force_trap": "Empty","ctlr_temp_number": 7,"ctlr_temp_status": "up","ctlr_temp_current_c": 31,"ctlr_temp_name": "BBU","index": "1.8"}]}]}}

尝试的JOLT配置（存在问题） #

[{"operation": "shift","spec": {"fragmented": "fragmented","nodeIp": "nodeIp","requestId": "requestId","nodeId": "nodeId","timestamp": "timestamp","requestTimestamp": "requestTimestamp","data": {"storage_controller": {"*": {"*": "data.storage_controller[&1].&","@(2,storage_controller_temp)": {"*": {"@": "data.storage_controller[&3].temp_sensors[]"}}}}}}]

问题：该配置将所有温度传感器数据添加到每个存储控制器的temp_sensors数组中，未根据index字段匹配对应的控制器。

正确的JOLT转换配置 #

[
  {
    "operation": "shift",
    "spec": {
      "fragmented": "fragmented",
      "nodeIp": "nodeIp",
      "requestId": "requestId",
      "nodeId": "nodeId",
      "timestamp": "timestamp",
      "requestTimestamp": "requestTimestamp",
      "data": {
        "storage_controller": "data.storage_controller",
        "storage_controller_temp": {
          "*": {
            "index": {
              "*.*": {
                "@2": "temp_groups.&1[]"
              }
            }
          }
        }
      }
    }
  },
  {
    "operation": "shift",
    "spec": {
      "fragmented": "fragmented",
      "nodeIp": "nodeIp",
      "requestId": "requestId",
      "nodeId": "nodeId",
      "timestamp": "timestamp",
      "requestTimestamp": "requestTimestamp",
      "data": {
        "storage_controller": {
          "*": {
            "*": "data.storage_controller[&1].&",
            "@(1,index)": {
              "@(3,temp_groups.&)": "data.storage_controller[&2].temp_sensors"
            }
          }
        }
      }
    }
  }
]

配置说明 #

1. 第一步Shift操作：

   • 保留顶层字段不变；
   • 拆分storage_controller_temp中每个对象的index字段，提取控制器编号作为分组键，将对应传感器数据存入temp_groups下的对应数组。

2. 第二步Shift操作：

   • 保留存储控制器的原有字段；
   • 根据每个存储控制器的index值，从temp_groups中取出对应分组的传感器数据，嵌入到该控制器的temp_sensors数组中。

内容的提问来源于stack exchange，提问作者Kamran Kamilli