无法使用recursivelySquashNulls时，基于id合并JSON数组的JOLT问题

需求说明 #

我被这个问题卡了好几周，之前需要合并JSON数据去重、生成更简洁的结构，但当前平台版本不支持recursivelySquashNulls参数，没这个参数的话现有JOLT配置会丢失格式。现在要实现的是：把clubhouse数组里相同id的对象合并，合并时保留nationalities数组结构，合并investors数组内容，其他字段取唯一值，最终得到指定的输出格式。

输入JSON

{
  "clubhouse": [
    {
      "id": "01",
      "statusId": "ok",
      "stateid": "2",
      "nationalities": [
        {
          "nationalityid": "1"
        },
        {
          "nationalityid": "2"
        },
        {
          "nationalityid": "3"
        }
      ],
      "TypeId": "3"
    },
    {
      "id": "01",
      "investors": [
        {
          "investor": {
            "id": "1234",
            "gender": "01"
          },
          "inamount": "1500000",
          "ratio": "12"
        }
      ]
    },
    {
      "id": "01",
      "investors": [
        {
          "investor": {
            "id": "4321",
            "gender": "02"
          },
          "inamount": "1700000",
          "ratio": "12"
        }
      ]
    },
    {
      "id": "02",
      "statusId": "ok",
      "stateid": "2",
      "nationalities": [
        {
          "nationalityid": "3"
        },
        {
          "nationalityid": "4"
        },
        {
          "nationalityid": "5"
        }
      ],
      "TypeId": "3"
    },
    {
      "id": "02",
      "investors": [
        {
          "investor": {
            "id": "1333",
            "gender": "01"
          },
          "inamount": "1500000",
          "ratio": "12"
        }
      ]
    },
    {
      "id": "03",
      "statusId": "ok",
      "stateid": "5",
      "nationalities": [
        {
          "nationalityid": "3"
        },
        {
          "nationalityid": "4"
        },
        {
          "nationalityid": "5"
        }
      ],
      "TypeId": "3"
    },
    {
      "id": "03",
      "investors": [
        {
          "investor": {
            "id": "",
            "gender": ""
          },
          "inamount": "",
          "ratio": ""
        }
      ]
    },
    {
      "id": "02",
      "statusId": "ok",
      "stateid": "2",
      "nationalities": [
        {
          "nationalityid": "3"
        },
        {
          "nationalityid": "4"
        },
        {
          "nationalityid": "5"
        }
      ],
      "TypeId": "3"
    },
    {
      "id": "02",
      "investors": [
        {
          "investor": {
            "id": "1334",
            "gender": "02"
          },
          "inamount": "1900000",
          "ratio": "12"
        }
      ]
    }
  ]
}

现有JOLT配置（无recursivelySquashNulls）

[
  {
    // group by "id" values to create separate objects 
    "operation": "shift",
    "spec": {
      "*": {
        "*": {
          "*": "@(1,id).&",
          "nationalities": {
            "*": {
              "@": "@(3,id).&2[&3][]"
            }
          },
          "investors": {
            "*": {
              "*": {
                "@": "@(4,id).&3[&4].&"
              }
            }
          }
        }
      }
    }
  },
  {
    // pick only the first components from the repeated values populated within the arrays 
    "operation": "cardinality",
    "spec": {
      "*": {
        "*": "ONE",
        "investors": "MANY",
        // Have to add nationalities here, if not it only shows null in the second array since the real values are hidden behind
        "nationalities": "MANY"
      }
    }
  },
  {
    // get rid of object labels
    "operation": "shift",
    "spec": {
      "*": ""
    }
  }
]

期望输出

{
  "clubhouse": [
    {
      "id": "01",
      "statusId": "ok",
      "stateid": "2",
            "nationalities": [
        {
          "nationalityid": "1"
        },
        {
          "nationalityid": "2"
        },
        {
          "nationalityid": "3"
        }
      ],
      "TypeId": "3",
      "investors": [
        {
          "investor": {
            "id": "1234",
            "gender": "01"
          },
          "inamount": "1500000",
          "ratio": "12"
        },
        {
          "investor": {
            "id": "4321",
            "gender": "02"
          },
          "inamount": "1700000",
          "ratio": "12"
        }
      ]
    },
    {
      "id": "02",
      "statusId": "ok",
      "stateid": "2",
      "nationalities": [
        {
          "nationalityid": "3"
        },
        {
          "nationalityid": "4"
        },
        {
          "nationalityid": "5"
        }
      ],
      "TypeId": "3",
      "investors": [
        {
          "investor": {
            "id": "1333",
            "gender": "01"
          },
          "inamount": "1500000",
          "ratio": "12"
        },
        {
          "investor": {
            "id": "1334",
            "gender": "02"
          },
          "inamount": "1900000",
          "ratio": "12"
        }
      ]
    },
    {
      "id": "03",
      "statusId": "ok",
      "stateid": "5",
      "nationalities": [
        {
          "nationalityid": "3"
        },
        {
          "nationalityid": "4"
        },
        {
          "nationalityid": "5"
        }
      ],
      "TypeId": "3",
      "investors": [
        {
          "investor": {
            "id": "",
            "gender": ""
          },
          "inamount": "",
          "ratio": ""
        }
      ]
    }
  ]
}

修正后的JOLT配置 #

[
  {
    "operation": "shift",
    "spec": {
      "clubhouse": {
        "*": {
          // 按id分组，将普通字段直接映射到对应id的对象下
          "id|statusId|stateid|TypeId": "@(1,id).&",
          // 保留nationalities数组结构，直接追加到对应id的nationalities数组中
          "nationalities": {
            "*": "@(3,id).nationalities[]"
          },
          // 合并investors数组内容，将每个investor对象追加到对应id的investors数组中
          "investors": {
            "*": "@(3,id).investors[]"
          }
        }
      }
    }
  },
  {
    "operation": "cardinality",
    "spec": {
      "*": {
        // 普通字段取唯一值
        "id|statusId|stateid|TypeId": "ONE",
        // nationalities和investors保留所有元素
        "nationalities": "MANY",
        "investors": "MANY"
      }
    }
  },
  {
    "operation": "shift",
    "spec": {
      // 将分组后的对象重新包装成clubhouse数组
      "*": "clubhouse[]"
    }
  },
  {
    "operation": "modify-overwrite-beta",
    "spec": {
      "clubhouse": {
        "*": {
          // 去重nationalities数组（根据nationalityid）
          "nationalities": "=distinct"
        }
      }
    }
  }
]

配置说明 #

1. 第一个shift操作：明确针对clubhouse数组处理，按id分组，普通字段直接映射，nationalities和investors数组元素分别追加到对应分组的数组中，避免结构混乱。
2. cardinality操作：指定普通字段取唯一值，数组类型字段保留所有元素，确保非数组字段不重复，数组内容完整。
3. 第二个shift操作：将分组后的对象重新整理为clubhouse数组，恢复期望的外层结构。
4. modify-overwrite-beta操作：对nationalities数组去重，避免相同nationalityid的元素重复出现。

内容的提问来源于stack exchange，提问作者Sinuers