如何通过Composite Aggregation结合TopHits聚合获取各用户最新文档

需求：

• 获取每个用户ID（influencerId）对应的最新文档

我尝试用Composite聚合遍历所有文档，但目前只能得到每个用户的文档计数doc_count，无法获取对应最新文档。最初的查询如下：

{
    "track_total_hits": false,
    "aggs": {
        "completions_users": {
            "composite": {
                "after": {
                    "influencerId": ""
                },
                "size": 10000,
                "sources": [
                    {
                        "influencerId": {
                            "terms": {
                                "field": "influencerId"
                            }
                        }
                    }
                ]
            }
        }
    },
    "query": {
        "bool": {
            "must": [
                {
                    "bool": {
                        "must": [
                            {
                                "terms": {
                                    "influencerId": [
                                        "XXXXX-ad84-4f35-8a58-9ee3cc8a3c6b",
                                        "YYYYYY-ad84-4f35-8a58-9ee3cc8a3c6b"
                                    ]
                                }
                            },
                            {
                                "match": {
                                    "campaignSponsorshipId": {
                                        "query": "XXXXXX-729e-4663-85f2-6ff3f986e93f"
                                    }
                                }
                            },
                            {
                                "match": {
                                    "status": {
                                        "query": "Completed"
                                    }
                                }
                            }
                        ]
                    }
                }
            ]
        }
    },
    "size": 0
}

该查询返回结果仅包含每个用户的文档计数：

{
    "took": 11,
    "timed_out": false,
    "_shards": {
        "total": 3,
        "successful": 3,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "max_score": null,
        "hits": []
    },
    "aggregations": {
        "completions_users": {
            "after_key": {
                "influencerId": "XXXXXX-ad84-4f35-8a58-9ee3cc8a3c6b"
            },
            "buckets": [
                {
                    "key": {
                        "influencerId": "XXXXXX-ad84-4f35-8a58-9ee3cc8a3c6b"
                    },
                    "doc_count": 6
                }
            ]
        }
    }
}

之后我尝试添加TopHits聚合，但错误地将其与Composite聚合平级，导致仅返回全局最新的1个文档，而非每个用户对应的最新文档：

{
    "track_total_hits": false,
    "aggs": {
        "search_last_completed": {
            "composite": {
                "after": {
                    "influencerId": ""
                },
                "size": 10000,
                "sources": [
                    {
                        "influencerId": {
                            "terms": {
                                "field": "influencerId"
                            }
                        }
                    }
                ]
            }
        },
        "most_recent_doc": {
            "top_hits": {
                "size": 1,
                "sort": [
                    {
                        "completedDate": {
                            "order": "desc"
                        }
                    }
                ],
                "_source": {
                    "includes": [
                        "completedDate",
                        "id",
                        "influencerId",
                        "campaignId",
                        "campaignSponsorshipSetId",
                        "campaignSponsorshipId"
                    ]
                }
            }
        }
    },
    "query": {
        "bool": {
            "must": [
                {
                    "bool": {
                        "must": [
                            {
                                "terms": {
                                    "influencerId": [
                                        "XXXXXX-85a2-40fa-9c88-f165f4685b73",
                                        "YYYYYY-85a2-40fa-9c88-f165f4685b73"
                                    ]
                                }
                            },
                            {
                                "match": {
                                    "status": {
                                        "query": "Completed"
                                    }
                                }
                            }
                        ]
                    }
                }
            ]
        }
    },
    "size": 0
}

返回结果不符合预期，仅得到1个全局最新文档：

{
    "took": 16,
    "timed_out": false,
    "_shards": {
        "total": 3,
        "successful": 3,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "max_score": null,
        "hits": []
    },
    "aggregations": {
        "most_recent_doc": { // 仅返回1个文档
            "hits": {
                "total": {
                    "value": 99,
                    "relation": "eq"
                },
                "max_score": null,
                "hits": [
                    {
                        "_index": "sponsorshipsinfluencers-v7-2022-8",
                        "_type": "_doc",
                        "_id": "a1ad8a13-eb82-4d9c-bd8b-de9ea03c6199",
                        "_score": null,
                        "_source": {
                            "campaignSponsorshipSetId": "XXXXXXX-c57a-487e-89b9-4d787c2dc778",
                            "influencerId": "XXXXXXX-85a2-40fa-9c88-f165f4685b73",
                            "campaignId": "XXXXX-d985-4aa7-bd18-e07e5988bb0a",
                            "campaignSponsorshipId": "XXXX-729e-4663-85f2-6ff3f986e93f",
                            "id": "XXXXX-eb82-4d9c-bd8b-de9ea03c6199",
                            "completedDate": "2022-08-08T12:03:52.9172233Z"
                        },
                        "sort": [
                            1659960232917
                        ]
                    }
                ]
            }
        },
        "search_last_completed": {
            "after_key": {
                "influencerId": "XXXXXX-85a2-40fa-9c88-f165f4685b73"
            },
            "buckets": [ // 每个Bucket仅包含计数，无文档内容
                {
                    "key": {
                        "influencerId": "XXXXX-85a2-40fa-9c88-f165f4685b73"
                    },
                    "doc_count": 99
                }
            ]
        }
    }
}

正确解决方案：嵌套TopHits聚合到Composite内部 #

要实现每个用户Bucket返回最新文档，需要将TopHits聚合作为Composite聚合的子聚合，这样每个用户Bucket都会执行一次TopHits查询，返回该用户的最新文档。正确的查询如下：

{
    "track_total_hits": false,
    "aggs": {
        "search_last_completed": {
            "composite": {
                "after": {
                    "influencerId": ""
                },
                "size": 10000,
                "sources": [
                    {
                        "influencerId": {
                            "terms": {
                                "field": "influencerId"
                            }
                        }
                    }
                ]
            },
            "aggs": {
                "most_recent_doc": {
                    "top_hits": {
                        "size": 1,
                        "sort": [
                            {
                                "completedDate": {
                                    "order": "desc"
                                }
                            }
                        ],
                        "_source": {
                            "includes": [
                                "completedDate",
                                "id",
                                "influencerId",
                                "campaignId",
                                "campaignSponsorshipSetId",
                                "campaignSponsorshipId"
                            ]
                        }
                    }
                }
            }
        }
    },
    "query": {
        "bool": {
            "must": [
                {
                    "bool": {
                        "must": [
                            {
                                "terms": {
                                    "influencerId": [
                                        "XXXXXX-85a2-40fa-9c88-f165f4685b73",
                                        "YYYYYY-85a2-40fa-9c88-f165f4685b73"
                                    ]
                                }
                            },
                            {
                                "match": {
                                    "status": {
                                        "query": "Completed"
                                    }
                                }
                            }
                        ]
                    }
                }
            ]
        }
    },
    "size": 0
}

预期返回结果结构 #

每个Composite Bucket中会包含对应的最新文档内容：

{
  "took": 16,
  "timed_out": false,
  "_shards": {
    "total": 3,
    "successful": 3,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "max_score": null,
    "hits": []
  },
  "aggregations": {
    "search_last_completed": {
      "after_key": {
        "influencerId": "XXXXXXX-85a2-40fa-9c88-f165f4685b73"
      },
      "buckets": [
        {
          "key": {
            "influencerId": "XXXXXXXX-85a2-40fa-9c88-f165f4685b73"
          },
          "doc_count": 99,
          "most_recent_doc": {
            "hits": {
              "hits": [
                {
                  "_source": {
                    "campaignSponsorshipSetId": "49ab4c80-c57a-487e-89b9-4d787c2dc778",
                    "influencerId": "XXXXXXXX-85a2-40fa-9c88-f165f4685b73",
                    "campaignId": "910330b8-d985-4aa7-bd18-e07e5988bb0a",
                    "campaignSponsorshipId": "47d2fc07-729e-4663-85f2-6ff3f986e93f",
                    "id": "a1ad8a13-eb82-4d9c-bd8b-de9ea03c6199",
                    "completedDate": "2022-08-08T12:03:52.9172233Z"
                  },
                  "sort": [1659960232917]
                }
              ]
            }
          }
        }
      ]
    }
  }
}

内容的提问来源于stack exchange，提问作者xplat