嵌套kNN搜索偶尔出现空inner hits的原因排查

在使用Elasticsearch嵌套kNN搜索时，偶现返回的文档没有任何inner hits的情况，相关细节如下：

搜索结果示例（注意最后两条结果的inner hits为空） #

{
  "_shards": {
    "failed": 0,
    "skipped": 0,
    "successful": 1,
    "total": 1
  },
  "hits": {
    "hits": [
      {
        "_id": "bRkrI4IBuKQL3UqO8DkV",
        "_index": "synthetic_data_index",
        "_score": 2.0406117,
        "_source": {
          "nested_object": {
            "cool_vector_field": [0.2234513587608724, 0.8878394741163076, 0.3087446303001422, 0.5401258921662346, -0.9228053400350715],
            "some_text_field": "This is nested text for doc number 383"
          },
          "non_nested_text": "This doc is number 383"
        },
        "inner_hits": {
          "nested_object": {
            "hits": {
              "hits": [
                {
                  "_id": "bRkrI4IBuKQL3UqO8DkV",
                  "_index": "synthetic_data_index",
                  "_nested": {
                    "field": "nested_object",
                    "offset": 0
                  },
                  "_score": 2.0406117,
                  "_source": {
                    "cool_vector_field": [0.2234513587608724, 0.8878394741163076, 0.3087446303001422, 0.5401258921662346, -0.9228053400350715],
                    "some_text_field": "This is nested text for doc number 383"
                  }
                }
              ],
              "max_score": 2.0406117,
              "total": {
                "relation": "eq",
                "value": 1
              }
            }
          }
        }
      },
      {
        "_id": "bhkrI4IBuKQL3UqO8DkV",
        "_index": "synthetic_data_index",
        "_score": 2.0406117,
        "_source": {
          "nested_object": {
            "cool_vector_field": [-0.3667193179233421, 0.04664013242577236, -0.4679759075333949, 0.9335512141017783, 0.9847209912260526],
            "some_text_field": "This is nested text for doc number 384"
          },
          "non_nested_text": "This doc is number 384"
        },
        "inner_hits": {
          "nested_object": {
            "hits": {
              "hits": [],
              "max_score": null,
              "total": {
                "relation": "eq",
                "value": 0
              }
            }
          }
        }
      },
      {
        "_id": "bxkrI4IBuKQL3UqO8DkV",
        "_index": "synthetic_data_index",
        "_score": 2.0406117,
        "_source": {
          "nested_object": {
            "cool_vector_field": [-0.9203098606975535, -0.8629298912981729, -0.4274567965220182, 0.5190442025173878, -0.32420767814040885],
            "some_text_field": "This is nested text for doc number 385"
          },
          "non_nested_text": "This doc is number 385"
        },
        "inner_hits": {
          "nested_object": {
            "hits": {
              "hits": [],
              "max_score": null,
              "total": {
                "relation": "eq",
                "value": 0
              }
            }
          }
        }
      }
    ],
    "max_score": 2.0406117,
    "total": {
      "relation": "gte",
      "value": 10000
    }
  },
  "status": 200,
  "timed_out": false,
  "took": 3
}

查询语句 #

{
  "query": {
    "nested": {
      "inner_hits": {},
      "path": "nested_object",
      "query": {
        "knn": {
          "nested_object.cool_vector_field": {
            "k": 3,
            "vector": [-0.53387915, -0.14078664, -0.41952186,  0.11891716, -0.30830444]
          }
        }
      },
      "score_mode": "max"
    }
  },
  "size": 3
}

索引设置 #

{
  "mappings": {
    "properties": {
      "nested_object": {
        "properties": {
          "cool_vector_field": {
            "dimension": 5,
            "method": {
              "engine": "nmslib",
              "name": "hnsw",
              "parameters": {
                "ef_construction": 128,
                "m": 24
              },
              "space_type": "innerproduct"
            },
            "type": "knn_vector"
          },
          "some_text_field": {
            "type": "text"
          }
        },
        "type": "nested"
      }
    }
  },
  "settings": {
    "index": {
      "knn": true,
      "knn.algo_param.ef_search": 100,
      "refresh_interval": "30s"
    },
    "number_of_shards": 1
  }
}

问题触发特征 #

该问题并非每次搜索都会出现，在文档量超过10000的数据集里更常见，且以下场景发生率更高：

• 文档数量越多
• 当size > k时
• 当size和k取值较大时

补充发现 #

进一步测试发现，单次查询中所有无inner hits的文档均为连续索引的文档组，例如：

['3395', '3396', '3397', '3398', '3399', '4250', '4251', '4252', '4253', '4254']

问题分析与建议 #

从现象和触发条件来看，这大概率是kNN插件的Bug。结合连续索引文档组出现问题的特征，推测可能是批量索引过程中向量索引的构建或分片内kNN检索的边界处理存在疏漏，导致部分连续写入的文档在嵌套kNN查询时无法匹配到对应的inner hits。

建议：

1. 升级kNN插件到最新稳定版本，确认是否已有官方修复方案；
2. 调整索引的kNN参数，比如增大ef_search值，或调整m、ef_construction参数，尝试缓解问题；
3. 向官方提交issue，提供完整的复现步骤、数据集特征和环境信息，帮助开发团队定位修复。

内容的提问来源于stack exchange，提问作者Bobby M