Logstash输出至Elasticsearch报400错误：索引模板安装失败求助

问题场景 #

使用logstash:8.5.3、elasticsearch:8.5.3、kibana:8.5.3搭建ELK栈抓取分析Twitter数据，启动时Logstash返回400错误，无法安装索引模板。

错误日志 #

Failed to install template {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://elasticsearch:9200/_index_template/twitter'", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:84:in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:324:in perform_request_to_url'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:311:in block in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:398:in with_connection'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:310:in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in block in Pool'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:412:in template_put'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/http_client.rb:85:in template_install'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:29:in install'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch/template_manager.rb:17:in install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch.rb:578:in install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch.rb:344:in finish_register'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/outputs/elasticsearch.rb:300:in block in register'", "/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-output-elasticsearch-11.9.3-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:154:in block in after_successful_connection'"]}

当前配置 #

索引模板（twitter.json） #

{
  "template": "twitter-*",
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "index.mapping.total_fields.limit": 2000
  },
  "mappings": {
    "_default_": {
      "_all": {
        "enabled": true
      },
      "properties": {
        "@timestamp": {
          "type": "date",
          "format": "dateOptionalTime"
        },
        "created_at": {
          "type": "date",
          "format": "EEE MMM dd HH:mm:ss Z YYYY"
        },
        "text": {
          "type": "text"
        },
        "user": {
          "type": "object",
          "properties": {
            "description": {
              "type": "text"
            }
          }
        },
        "coordinates": {
          "type": "object",
          "properties": {
            "coordinates": {
              "type": "geo_point"
            }
          }
        },
        "entities": {
          "type": "object",
          "properties": {
            "hashtags": {
              "type": "object",
              "properties": {
                "text": {
                  "type": "text",
                  "fielddata": true
                }
              }
            }
          }
        },
        "retweeted_status": {
          "type": "object",
          "properties": {
            "text": {
              "type": "text"
            }
          }
        }
      },
      "dynamic_templates": [
        {
          "string_template": {
            "match": "*",
            "match_mapping_type": "string",
            "mapping": {
              "type": "keyword"
            }
          }
        }
      ]
    }
  }
}

Logstash配置 #

input {
        tcp {
                port => 50000
        }
}

filter {

}

output {
        elasticsearch {
                hosts => "elasticsearch:9200"
                user => "elastic"
                password => "${LOGSTASH_INTERNAL_PASSWORD}"
                index => "twitter-%{+yyyy.MM.dd}"
                document_type => "tweets"
                template      => "./templates/twitter.json"
                template_name => "twitter"
                template_overwrite => true
        }
}

问题原因及修复方案 #

你的模板和Logstash配置存在多个与Elasticsearch 8.x不兼容的点：

1. 索引模板格式过时：Elasticsearch 8.x使用索引模板v2格式，原模板中的template字段需替换为index_patterns，且不再支持_default_映射类型。
2. _all字段已移除：Elasticsearch 6.x之后就废弃了_all字段，8.x完全不支持该配置。
3. document_type已废弃：Elasticsearch 7.x开始移除了文档类型，Logstash输出中document_type字段需删除。
4. 日期格式问题：dateOptionalTime是旧格式，8.x推荐使用strict_date_optional_time；created_at的格式中YYYY需改为yyyy（小写y代表年）。
5. 动态模板语法：match_mapping_type: string在8.x中已废弃，需改为match_mapping_type: text。

修复后的索引模板 #

{
  "index_patterns": "twitter-*",
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "index.mapping.total_fields.limit": 2000
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date",
        "format": "strict_date_optional_time"
      },
      "created_at": {
        "type": "date",
        "format": "EEE MMM dd HH:mm:ss Z yyyy"
      },
      "text": {
        "type": "text"
      },
      "user": {
        "type": "object",
        "properties": {
          "description": {
            "type": "text"
          }
        }
      },
      "coordinates": {
        "type": "object",
        "properties": {
          "coordinates": {
            "type": "geo_point"
          }
        }
      },
      "entities": {
        "type": "object",
        "properties": {
          "hashtags": {
            "type": "nested",
            "properties": {
              "text": {
                "type": "text",
                "fielddata": true
              }
            }
          }
        }
      },
      "retweeted_status": {
        "type": "object",
        "properties": {
          "text": {
            "type": "text"
          }
        }
      }
    },
    "dynamic_templates": [
      {
        "string_to_keyword": {
          "match": "*",
          "match_mapping_type": "text",
          "mapping": {
            "type": "keyword"
          }
        }
      }
    ]
  }
}

修复后的Logstash输出配置 #

output {
        elasticsearch {
                hosts => "elasticsearch:9200"
                user => "elastic"
                password => "${LOGSTASH_INTERNAL_PASSWORD}"
                index => "twitter-%{+yyyy.MM.dd}"
                template      => "./templates/twitter.json"
                template_name => "twitter"
                template_overwrite => true
        }
}

额外验证步骤 #

• 手动测试模板是否能在Elasticsearch中创建：

  curl -X PUT "http://elasticsearch:9200/_index_template/twitter" -u elastic:${LOGSTASH_INTERNAL_PASSWORD} -H "Content-Type: application/json" -d @./templates/twitter.json
  

  若返回错误，Elasticsearch会给出具体的字段错误提示，可根据提示进一步调整。

内容的提问来源于stack exchange，提问作者Sacha Athias