Elasticsearch 5.6升级至8.9后_score差异及结果排序问题排查

问题描述 #

将Elasticsearch从5.6版本升级至8.9版本后，原有按weight字段和_score排序的查询出现结果排序变化，核心原因是升级后文档的_score值与旧版本存在显著差异。

查询语句 #

POST /auto-complete/_search?typed_keys=true
{
    "size": 5,
    "query": {
        "bool": {
            "should": [
                {
                    "match_phrase_prefix": {
                        "suggestion": {
                            "query": "the"
                        }
                    }
                },
                {
                    "match": {
                        "suggestion.analyzed": {
                            "fuzziness": "AUTO",
                            "operator": "and",
                            "query": "the"
                        }
                    }
                }
            ]
        }
    },
    "sort": [
        {
            "weight": {
                "order": "desc"
            }
        },
        {
            "_score": {
                "order": "desc"
            }
        }
    ]
}

5.6版本查询结果 #

{
  "took": 129,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 4858,
    "max_score": null,
    "hits": [
      {
        "_index": "auto-complete",
        "_type": "default",
        "_id": "The Thelma Hoop Earrings",
        "_score": 6.3522644,
        "_source": {
          "suggestion": "The Thelma Hoop Earrings",
          "weight": 1
        },
        "sort": [
          1,
          6.3522644
        ]
      },
      {
        "_index": "auto-complete",
        "_type": "default",
        "_id": "The Theresa Ring",
        "_score": 6.3522644,
        "_source": {
          "suggestion": "The Theresa Ring",
          "weight": 1
        },
        "sort": [
          1,
          6.3522644
        ]
      },
      {
        "_index": "auto-complete",
        "_type": "default",
        "_id": "The Theodora Ring",
        "_score": 6.337865,
        "_source": {
          "suggestion": "The Theodora Ring",
          "weight": 1
        },
        "sort": [
          1,
          6.337865
        ]
      },
      {
        "_index": "auto-complete",
        "_type": "default",
        "_id": "The Thea Ring",
        "_score": 6.337865,
        "_source": {
          "suggestion": "The Thea Ring",
          "weight": 1
        },
        "sort": [
          1,
          6.337865
        ]
      },
      {
        "_index": "auto-complete",
        "_type": "default",
        "_id": "The Theor Band For Him",
        "_score": 5.7033815,
        "_source": {
          "suggestion": "The Theor Band For Him",
          "weight": 1
        },
        "sort": [
          1,
          5.7033815
        ]
      }
    ]
  }
}

8.9版本查询结果 #

{
  "took": 28,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 4874,
      "relation": "eq"
    },
    "max_score": null,
    "hits": [
      {
        "_index": "auto-complete",
        "_id": "The Theodora Ring",
        "_score": 8.927014,
        "_source": {
          "suggestion": "The Theodora Ring",
          "weight": 1
        },
        "sort": [
          1,
          8.927014
        ]
      },
      {
        "_index": "auto-complete",
        "_id": "The Theresa Ring",
        "_score": 8.927014,
        "_source": {
          "suggestion": "The Theresa Ring",
          "weight": 1
        },
        "sort": [
          1,
          8.927014
        ]
      },
      {
        "_index": "auto-complete",
        "_id": "The Thea Ring",
        "_score": 8.927014,
        "_source": {
          "suggestion": "The Thea Ring",
          "weight": 1
        },
        "sort": [
          1,
          8.927014
        ]
      },
      {
        "_index": "auto-complete",
        "_id": "The Thelma Hoop Earrings",
        "_score": 7.9907713,
        "_source": {
          "suggestion": "The Thelma Hoop Earrings",
          "weight": 1
        },
        "sort": [
          1,
          7.9907713
        ]
      }
    ]
  }
}

5.6版本映射配置 #

curl -X PUT "localhost:9201/auto-complete?pretty" -H 'Content-Type: application/json' -d'
{ 
"mappings" : 
{
  "default": {
    "properties": {
      "suggestion": {
        "type": "text",
        "fields": {
          "analyzed": {
            "type": "text",
            "analyzer": "nGram_analyzer",
            "search_analyzer": "whitespace"
          }
        }
      },
      "weight": {
        "type": "integer"
      }
    }
  }
}, 
"settings" : 
{
  "number_of_shards": 1,
  "number_of_replicas": 1,
  "index": {
    "analysis": {
      "analyzer": {
        "nGram_analyzer": {
          "type": "custom",
          "tokenizer": "whitespace",
          "filter": [
            "lowercase",
            "asciifolding",
            "nGram_filter"
          ]
        }
      },
      "filter": {
        "nGram_filter": {
          "type": "nGram",
          "min_gram": 2,
          "max_gram": 20,
          "token_chars": [
            "letter",
            "digit",
            "punctuation",
            "symbol"
          ]
        }
      }
    }
  }
}
}'

8.9版本映射配置 #

curl -X PUT "localhost:9201/auto-complete?pretty" -H 'Content-Type: application/json' -d'
{ 
"mappings" : 
{
  "properties": {
    "suggestion": {
      "type": "text",
      "fields": {
        "analyzed": {
          "type": "text",
          "analyzer": "nGram_analyzer",
          "search_analyzer": "whitespace"
        }
      }
    },
    "weight": {
      "type": "integer"
    }
  }
}, 
"settings" : 
{
  "number_of_shards": 1,
  "number_of_replicas": 1,
  "max_ngram_diff" : 18,
  "index": {
    "analysis": {
      "analyzer": {
        "nGram_analyzer": {
          "type": "custom",
          "tokenizer": "whitespace",
          "filter": [
            "lowercase",
            "asciifolding",
            "nGram_filter"
          ]
        }
      },
      "filter": {
        "nGram_filter": {
          "type": "ngram",
          "min_gram": 2,
          "max_gram": 20,
          "token_chars": [
            "letter",
            "digit",
            "punctuation",
            "symbol"
          ]
        }
      }
    }
  }
}} 
'

问题原因分析 #

1. 评分算法细节变更：从5.6到8.9，Elasticsearch对match_phrase_prefix和bool should的评分计算逻辑做了优化调整，包括前缀匹配的权重分配、多子句评分的合并方式，导致最终_score值变化。
2. 分片数不一致：5.6版本查询结果显示索引使用了5个分片，而8.9版本配置为1个分片。分片数不同会影响IDF（逆文档频率）的计算，进而改变_score值。
3. 版本间的隐性优化：ES在后续版本中对BM25相似度算法的参数默认值、词频统计逻辑等做了细微调整，这些变化都会反映在最终的评分结果上。

解决方案 #

1. 统一索引分片数 #

将8.9版本的索引分片数调整为与5.6版本一致（5个分片），确保IDF计算的统计基数相同，减少评分差异。注：分片数无法直接修改，需重新创建索引并迁移数据，创建时指定分片数：

PUT /auto-complete
{
  "settings": {
    "number_of_shards": 5,
    // 其他原有配置
  },
  "mappings": {
    // 原有映射配置
  }
}

2. 使用自定义评分逻辑固定排序 #

通过function_score自定义评分规则，避免依赖ES版本默认的_score计算。例如给两个should子句分配固定权重，或直接复用旧版本的评分比例：

POST /auto-complete/_search?typed_keys=true
{
    "size": 5,
    "query": {
        "function_score": {
            "query": {
                "bool": {
                    "should": [
                        {
                            "match_phrase_prefix": {
                                "suggestion": {
                                    "query": "the"
                                }
                            }
                        },
                        {
                            "match": {
                                "suggestion.analyzed": {
                                    "fuzziness": "AUTO",
                                    "operator": "and",
                                    "query": "the"
                                }
                            }
                        }
                    ]
                }
            },
            "functions": [
                {
                    "filter": {"match_phrase_prefix": {"suggestion": "the"}},
                    "weight": 0.6 // 根据旧版本评分占比调整
                },
                {
                    "filter": {"match": {"suggestion.analyzed": "the"}},
                    "weight": 0.4
                }
            ],
            "score_mode": "sum"
        }
    },
    "sort": [
        {
            "weight": {
                "order": "desc"
            }
        },
        {
            "_score": {
                "order": "desc"
            }
        }
    ]
}

3. 显式配置BM25参数对齐旧版本 #

在8.9版本索引中显式配置与5.6版本一致的BM25参数，确保相似度计算逻辑一致：

PUT /auto-complete/_settings
{
  "index.similarity.default.type": "BM25",
  "index.similarity.default.k1": 1.2,
  "index.similarity.default.b": 0.75
}

4. 利用explain排查具体差异 #

在查询中添加explain: true参数，查看每个文档的评分计算步骤，对比5.6和8.9版本的差异点，针对性调整：

POST /auto-complete/_search?typed_keys=true&explain=true
{
    // 原有查询内容
}

内容的提问来源于stack exchange，提问作者Rohit Gaur