ReactJS数据表格Elasticsearch过滤后无数据问题排查求助

问题现象 #

使用React实现的数据表格通过Node.js后端从Elasticsearch拉取数据填充，未应用过滤条件时数据检索和展示均正常；添加过滤条件后，尽管Elasticsearch的_source中存在匹配记录，数据表格仍显示为空。

相关信息 #

请求参数结构 #

{
  "pageIndex": 1,
  "pageSize": 10,
  "sort": { "order": "", "key": "" },
  "query": "",
  "filterData": {
    "analysis": [ "0", "1", "2", "3" ],
    "threat_level_id": [ "1", "2", "3", "4" ]
  }
}

接口地址 #

POST /api/v1/events/public/list

带过滤的Node.js控制器代码 #

exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query, filterData } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          bool: {
            must: [
              {
                match_all: {},
              },
            ],
            filter: [],
          },
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query.bool.must = [
        {
          match: {
            'Event.info': {
              query: query,
              fuzziness: 'AUTO',
            },
          },
        },
      ]
    }
    if (filterData.analysis.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.analysis': filterData.analysis,
        },
      })
    }
    if (filterData.threat_level_id.length > 0) {
      esQuery.body.query.bool.filter.push({
        terms: {
          'Event.threat_level_id': filterData.threat_level_id,
        },
      })
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

无过滤的Node.js控制器代码（可正常运行） #

exports.getPublicEvents = async (req, res) => {
  try {
    client.ping()
    const { pageIndex, pageSize, sort, query } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          match_all: {},
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query = {
        match: {
          'Event.info': {
            query: query,
            fuzziness: 'AUTO',
          },
        },
      }
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

Elasticsearch版本 #

7.17.8

生成的esQuery日志 #

{
  "index": "INDEX_NAME",
  "body": {
    "query": {
      "bool": {
        "must": [{ "match_all": {} }],
        "filter": [
          { "terms": { "Event.analysis": ["0", "1", "2"] } },
          { "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
        ]
      }
    },
    "from": 0,
    "size": 10
  }
}

Elasticsearch中的数据结构 #

{
    "@version": "1",
    "@timestamp": "2023-02-01T14:43:09.997Z",
    "Event": {
        "info": ".......................",
        "description": ".......................",
        "analysis": 0,
        "threat_level_id": "4",
        "created_at": 1516566351,
        "uuid": "5a64f74f0e543738c12bc973322",
        "updated_at": 1675262417
    }
}

索引映射 #

{
    "index_patterns": ["INDEX_NAME"],
    "template": "TEMPLATE_NAME",
    "settings": {
      "number_of_replicas": 0,
      "index.mapping.nested_objects.limit": 10000000
      },
    "mappings": {
      "dynamic": false,
      "properties": {
          "@timestamp": {
          "type": "date"
        },
        "Event": {
          "type": "nested",
          "properties": {
            "date_occured": {
              "type": "date"
            },
            "threat_level_id": {
              "type": "integer"
            },
            "description": {
              "type": "text"
            },
            "is_shared": {
              "type": "boolean"
            },
            "analysis": {
              "type": "integer"
            },
            "uuid": {
              "type": "text"
            },
            "created_at": {
              "type": "date"
            },
            "info": {
              "type": "text"
            },
            "shared_with": {
                "type": "nested",
                 "properties": {
                  "_id": {
                    "type": "text"
                }
              }
            },
            "updated_at": {
              "type": "date"
            },
            "author": {
              "type": "text"
            },
            "Attributes": {
              "type": "nested",
              "properties": {
                "data": {
                  "type": "text"
                },
                "type": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                },
                "comment": {
                  "type": "text"
                },
                "category": {
                  "type": "text"
                },
                "value": {
                  "type": "text"
                },
                "timestamp": {
                  "type": "date"
                }
              }
            }, 
            "organisation": {
              "type": "nested",
              "properties": {
                "name": {
                  "type": "text"
                },
                "uuid": {
                  "type": "text"
                }
              }
            },
            "Tags": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            },
            "TLP": {
              "type": "nested",
              "properties": {
                "color": {
                  "type": "text"
                },
                "name": {
                  "type": "text"
                }
              }
            }
          }
        }  
      }
    }
  }

问题根源与修复方案 #

问题1：Nested类型字段查询方式错误 #

从索引映射可知，Event字段是nested类型。Elasticsearch中nested类型字段需使用nested查询语法才能正确匹配，普通terms查询无法穿透nested结构，导致无匹配结果。

问题2：参数类型不匹配 #

请求中filterData的analysis和threat_level_id是字符串数组，但索引映射中这两个字段类型为integer，字符串与数字类型不匹配，terms查询无法匹配对应数据。

修复后的控制器代码 #

将过滤部分改为nested查询，并将参数转换为数字类型：

exports.getPublicEvents = async (req, res) => {
  try {
    await client.ping()
    const { pageIndex, pageSize, sort, query, filterData } = req.body
    let esQuery = {
      index: 'ns_*',
      body: {
        query: {
          bool: {
            must: [
              {
                match_all: {},
              },
            ],
            filter: [],
          },
        },
        from: (pageIndex - 1) * pageSize,
        size: pageSize,
      },
    }
    if (query) {
      esQuery.body.query.bool.must = [
        {
          nested: {
            path: 'Event',
            query: {
              match: {
                'Event.info': {
                  query: query,
                  fuzziness: 'AUTO',
                }
              }
            }
          }
        },
      ]
    }
    // 处理analysis过滤：转数字+嵌套查询
    if (filterData.analysis.length > 0) {
      const analysisValues = filterData.analysis.map(val => parseInt(val, 10))
      esQuery.body.query.bool.filter.push({
        nested: {
          path: 'Event',
          query: {
            terms: {
              'Event.analysis': analysisValues
            }
          }
        }
      })
    }
    // 处理threat_level_id过滤：转数字+嵌套查询
    if (filterData.threat_level_id.length > 0) {
      const threatLevelValues = filterData.threat_level_id.map(val => parseInt(val, 10))
      esQuery.body.query.bool.filter.push({
        nested: {
          path: 'Event',
          query: {
            terms: {
              'Event.threat_level_id': threatLevelValues
            }
          }
        }
      })
    }
    let esResponse = await client.search(esQuery)
    let data = esResponse.hits.hits.map((hit) => hit._source)
    let total = esResponse.hits.total.value

    res.status(200).json({
      status: 'success',
      data: data,
      total: total,
    })
  } catch (error) {
    res.status(500).json({
      error: 'Error connecting to Elasticsearch',
      errorMessage: error.message,
    })
  }
}

额外注意点 #

• 原代码中client.ping()未加await，会导致ping操作异步执行，无法确保ES连接正常，修复时添加了await。
• 查询Event.info时，因Event是nested类型，同样需要使用nested查询语法，否则也可能出现查询不到数据的情况。

内容的提问来源于stack exchange，提问作者Richard Branson