Elasticsearch双层嵌套字段聚合：仅获取匹配分类的实体

我有一个用于测试的Elasticsearch索引，其映射包含两层嵌套字段：

索引映射 #

test/
{
  "mappings": {
    "dynamic": "strict",
    "properties": {
      "observation_session_id": {
        "type": "long"
      },
      "hazard_categories": {
        "type": "nested",
        "include_in_root": true,
        "properties": {
          "sentence": {
            "type": "keyword"
          },
          "entities": {
            "type": "nested",
            "include_in_root": true,
            "properties": {
              "key": {
                "type": "keyword"
              },
              "value": {
                "type": "byte"
              }
            }
          },
          "category": {
            "type": "nested",
            "include_in_root": true,
            "properties": {
              "key": {
                "type": "keyword"
              },
              "value": {
                "type": "byte"
              }
            }
          }
        }
      }
    }
  }
}

已写入的测试文档 #

test/_doc/1
{
  "observation_session_id": 1,
  "hazard_categories": [{
    "sentence": "Test Sentence 1",
    "entities": [
      {
        "key": "spilled",
        "value": 1
      },
      {
        "key": "clean up",
        "value": 1
      }
    ],
    "category": [
      {
        "key": "Housekeeping",
        "value": 1
      },
      {
        "key": "Controls",
        "value": 1
      }
    ]
  },
 {
    "sentence": "Test Sentence 1.1",
    "category": [
      {
        "key": "PPE",
        "value": 1
      },
      {
        "key": "Housekeeping",
        "value": 1
      }
    ]
  }]
}

test/_doc/2
{
  "observation_session_id": 2,
  "hazard_categories": [{
    "sentence": "Test Sentence 2",
    "entities": [
      {
        "key": "spilled",
        "value": 1
      }
    ],
    "category": [
      {
        "key": "Housekeeping",
        "value": 1
      }
    ]
  },
 {
    "sentence": "Test Sentence 2.1",
    "entities": [
      {
        "key": "monitoring",
        "value": 1
      }
    ],
    "category": [
      {
        "key": "Controls",
        "value": 1
      }
    ]
  },
  {
    "sentence": "Test Sentence 2.2",
    "entities": [
      {
        "key": "gloves",
        "value": 1
      },
       {
        "key": "ppe kit",
        "value": 1
      }
    ],
    "category": [
      {
        "key": "PPE",
        "value": 1
      }
    ]
  }]
}

test/_doc/3
{
  "observation_session_id": 3,
  "hazard_categories": [{
    "sentence": "Test Sentence 3",
   
    "category": [
      {
        "key": "Housekeeping",
        "value": 1
      },
      {
        "key": "Controls",
        "value": 1
      },
      {
        "key": "PPE",
        "value": 1
      },
      {
        "key": "Care",
        "value": 1
      }
    ]
  }]
}

/test/_doc/4
{
  "observation_session_id": 4,
  "hazard_categories": [{
    "sentence": "Test Sentence 4",
    "entities": [
      {
        "key": "cover",
        "value": 1
      },
       {
        "key": "wire",
        "value": 1
      }
    ],
    "category": [
      {
        "key": "Mobile Equipment",
        "value": 1
      },
      {
        "key": "Electricals",
        "value": 1
      },
      {
        "key": "Lifting",
        "value": 1
      }
    ]
  }]
}

我的需求是获取匹配指定分类（如Housekeeping）的entities列表，但当前执行的查询返回了同一文档中不属于匹配分类的实体：

当前查询 #

{
  "query": {
    "nested": {
      "path": "hazard_categories",
      "query": {
        "bool": {
          "must": [
            {
              "nested": {
                "path": "hazard_categories.category",
                "query": {
                  "bool": {
                    "must": [
                      {
                        "match": {
                          "hazard_categories.category.key": "Housekeeping"
                        }
                      }
                    ]
                  }
                }
              }
            }
          ]
        }
      }
    }
  },
  "aggs": {
    "hazard_categories": {
      "nested": {
        "path": "hazard_categories.entities"
      },
      "aggs": {
        "total_entities": {
          "terms": {
            "field": "hazard_categories.entities.key"
          }
        }
      }
    }
  }
}

当前返回结果 #

{
  "buckets": [
    {
      "key": "spilled",
      "doc_count": 2
    },
    {
      "key": "clean up",
      "doc_count": 1
    },
    {
      "key": "gloves",
      "doc_count": 1
    },
    {
      "key": "monitoring",
      "doc_count": 1
    },
    {
      "key": "ppe kit",
      "doc_count": 1
    }
  ]
}

期望结果 #

{
  "buckets": [
    {
      "key": "spilled",
      "doc_count": 2
    },
    {
      "key": "clean up",
      "doc_count": 1
    }
  ]
}

解决方案 #

原查询的问题在于：虽然通过nested查询过滤出了包含Housekeeping分类的文档，但聚合时会统计整个文档内所有hazard_categories下的entities，包括同一文档中其他分类对应的实体。

需要在聚合阶段先过滤出匹配目标分类的hazard_categories条目，再对其中的entities进行统计，正确的查询如下：

{
  "query": {
    "nested": {
      "path": "hazard_categories",
      "query": {
        "bool": {
          "must": [
            {
              "nested": {
                "path": "hazard_categories.category",
                "query": {
                  "match": {
                    "hazard_categories.category.key": "Housekeeping"
                  }
                }
              }
            }
          ]
        }
      }
    }
  },
  "aggs": {
    "filtered_hazard_categories": {
      "nested": {
        "path": "hazard_categories"
      },
      "aggs": {
        "has_housekeeping": {
          "filter": {
            "nested": {
              "path": "hazard_categories.category",
              "query": {
                "match": {
                  "hazard_categories.category.key": "Housekeeping"
                }
              }
            }
          },
          "aggs": {
            "entities": {
              "nested": {
                "path": "hazard_categories.entities"
              },
              "aggs": {
                "total_entities": {
                  "terms": {
                    "field": "hazard_categories.entities.key"
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

逻辑说明 #

1. 首先通过外层nested查询过滤出包含Housekeeping分类的文档；
2. 在聚合阶段，先进入hazard_categories嵌套层级，添加filter聚合筛选出当前hazard_categories条目下包含Housekeeping分类的内容；
3. 仅对筛选后的hazard_categories条目内的entities进行嵌套聚合和terms统计，确保只统计匹配分类对应的实体。

内容的提问来源于stack exchange，提问作者sjsj15