如何用Python在Elastic中创建告警索引并编写数据对比告警函数

1. 创建Elastic告警索引的Python函数 #

首先安装Elasticsearch官方Python客户端：

pip install elasticsearch

编写创建告警索引的函数，包含字段映射定义：

from elasticsearch import Elasticsearch
from datetime import datetime

def create_alert_index(es_client, index_name="alerts-index"):
    # 检查索引是否已存在
    if not es_client.indices.exists(index=index_name):
        index_mapping = {
            "mappings": {
                "properties": {
                    "indexname": {"type": "text"},
                    "presentday": {"type": "integer"},
                    "previousday": {"type": "integer"},
                    "difference_percent": {"type": "float"},
                    "alert_type": {"type": "keyword"},
                    "timestamp": {"type": "date"}
                }
            }
        }
        es_client.indices.create(index=index_name, body=index_mapping)
        print(f"告警索引 {index_name} 创建成功")
    else:
        print(f"告警索引 {index_name} 已存在")

2. 数据对比与告警触发函数 #

编写处理数据、校验告警条件并将结果写入Elastic的函数：

def process_index_alerts(es_client, data, index_name="alerts-index"):
    alerts = []
    for entry in data:
        indexname = entry["indexname"]
        present = entry["presentday"]
        previous = entry["previousday"]
        
        # 处理前日数据为0的特殊情况
        if previous == 0:
            if present == 0:
                # 两日数据均为0，触发无变化告警
                alerts.append({
                    "indexname": indexname,
                    "presentday": present,
                    "previousday": previous,
                    "difference_percent": 0.0,
                    "alert_type": "no_change",
                    "timestamp": datetime.utcnow().isoformat()
                })
            else:
                # 前日为0当日非0，触发显著变化告警
                alerts.append({
                    "indexname": indexname,
                    "presentday": present,
                    "previousday": previous,
                    "difference_percent": 100.0,
                    "alert_type": "significant_change",
                    "timestamp": datetime.utcnow().isoformat()
                })
        else:
            # 计算变化百分比
            diff_percent = ((present - previous) / previous) * 100
            # 校验告警条件：差值为0 或 变化率绝对值≥20%
            if present == previous or abs(diff_percent) >= 20:
                alert_type = "no_change" if present == previous else "significant_change"
                alerts.append({
                    "indexname": indexname,
                    "presentday": present,
                    "previousday": previous,
                    "difference_percent": round(diff_percent, 2),
                    "alert_type": alert_type,
                    "timestamp": datetime.utcnow().isoformat()
                })
    
    # 批量写入告警到Elastic
    if alerts:
        bulk_ops = []
        for alert in alerts:
            bulk_ops.append({"index": {"_index": index_name}})
            bulk_ops.append(alert)
        es_client.bulk(body=bulk_ops)
        print(f"已触发 {len(alerts)} 条告警并写入Elastic")
    else:
        print("未触发任何告警")

3. 完整使用示例 #

将函数与你的数据结合运行：

# 初始化Elasticsearch客户端（根据你的集群配置修改）
es = Elasticsearch(
    ["http://localhost:9200"],
    basic_auth=("your_username", "your_password")  # 无需认证可删除此行
)

# 创建告警索引
create_alert_index(es)

# 你的原始数据
data = [{"indexname": "awsbill-octopusenabled*", "presentday": 301, "previousday": 301}, {"indexname": "awsbill-octopusrole*", "presentday": 335, "previousday": 334}, {"indexname": "awsbill-usernrole*", "presentday": 279, "previousday": 279}, {"indexname": "awsbill-gc-rbac*", "presentday": 3914, "previousday": 3917}, {"indexname": "awsbill-awsallusernrole*", "presentday": 235, "previousday": 234}, {"indexname": "awsbill_ec2volupd*", "presentday": 31911, "previousday": 32010}, {"indexname": "awsbill-predicted-ec2vol*", "presentday": 5649, "previousday": 4826}, {"indexname": "awsbill-isdsaws-accounts*", "presentday": 6026, "previousday": 6026}, {"indexname": "awsbill-users-account-info*", "presentday": 11236, "previousday": 11222}, {"indexname": "awsbill-config*", "presentday": 101736, "previousday": 101808}, {"indexname": "awsbill-budgets*", "presentday": 568, "previousday": 568}, {"indexname": "awsbill-cases*", "presentday": 35, "previousday": 36}, {"indexname": "awsbill-cost-usage*", "presentday": 11279, "previousday": 11654}, {"indexname": "awsbill_s3_metrics_bsize*", "presentday": 698, "previousday": 698}, {"indexname": "awsbill_s3api_list_objects*", "presentday": 954, "previousday": 954}, {"indexname": "awsbill_s3api_object_versions*", "presentday": 954, "previousday": 954}, {"indexname": "awsbill-ec2*", "presentday": 11963, "previousday": 12260}, {"indexname": "awsbill_ec2upd*", "presentday": 9092, "previousday": 9300}, {"indexname": "awsbill-elasticcache*", "presentday": 241, "previousday": 241}, {"indexname": "awsbill-iam-policies*", "presentday": 5244, "previousday": 5239}, {"indexname": "awsbill-iam-roles*", "presentday": 49473, "previousday": 49468}, {"indexname": "awsbill-images*", "presentday": 571712, "previousday": 572146}, {"indexname": "awsbill-kms*", "presentday": 9373, "previousday": 9301}, {"indexname": "awsbill-rds*", "presentday": 2294, "previousday": 2280}, {"indexname": "awsbill_rdsupd*", "presentday": 2294, "previousday": 2280}, {"indexname": "awsbill-s3*", "presentday": 4563, "previousday": 4555}, {"indexname": "awsbill-ssm*", "presentday": 122475, "previousday": 122475}, {"indexname": "awsbill-support*", "presentday": 12458, "previousday": 12562}, {"indexname": "awsbill-vol*", "presentday": 39673, "previousday": 39723}, {"indexname": "awsbill_ctrail*", "presentday": 189, "previousday": 268}, {"indexname": "awsbill-health*", "presentday": 66583, "previousday": 65313}, {"indexname": "awsbill_costusage*", "presentday": 0, "previousday": 14120}, {"indexname": "awsbill_eks_cluster*", "presentday": 111, "previousday": 111}, {"indexname": "awsbill_eks_nodes*", "presentday": 0, "previousday": 0}, {"indexname": "awsbill_eks_namespaces*", "presentday": 0, "previousday": 0}, {"indexname": "awsbill_eks_pods*", "presentday": 0, "previousday": 0}, {"indexname": "awsbill-grand-central*", "presentday": 6026, "previousday": 6026}, {"indexname": "gcpbill_disks*", "presentday": 14, "previousday": 14}, {"indexname": "gcpbill_iam_roles*", "presentday": 1097, "previousday": 1097}, {"indexname": "gcpbill_instances*", "presentday": 4, "previousday": 4}, {"indexname": "agg-awsbill-cwperc-cpuutilization-v2*", "presentday": 6848, "previousday": 6640}, {"indexname": "agg-awsbill-cwperc-mem-cached-v2*", "presentday": 5866, "previousday": 5256}, {"indexname": "agg-awsbill-cwperc-mem-total-v2*", "presentday": 2322, "previousday": 2859}, {"indexname": "agg-awsbill-cwperc-mem-used-v2*", "presentday": 462, "previousday": 882}, {"indexname": "agg-awsbill-cwperc-volumereadops*", "presentday": 0, "previousday": 0}, {"indexname": "agg-awsbill-cwperc-volumewriteops*", "presentday": 0, "previousday": 0}, {"indexname": "rollup-awsbill-cwperc-cpuutilization-p99-v2*", "presentday": 11593, "previousday": 11423}, {"indexname": "rollup-awsbill-cwperc-mem-cached-p99-v2*", "presentday": 5355, "previousday": 5260}, {"indexname": "rollup-awsbill-cwperc-mem-total-p99-v2*", "presentday": 2163, "previousday": 2094}, {"indexname": "rollup-awsbill-cwperc-mem-used-p99-v2*", "presentday": 871, "previousday": 871}, {"indexname": "rollup-awsbill-cwvol-volumereadops-v3*", "presentday": 0, "previousday": 0}, {"indexname": "rollup-awsbill-cwvol-awsbill_cwvol_volumewriteops-v3*", "presentday": 0, "previousday": 0}]

# 处理数据并触发告警
process_index_alerts(es, data)

核心逻辑说明 #

• 告警触发规则：
  1. 当日与前日数据完全一致（差值为0）
  2. 数据变化率绝对值达到或超过20%
  3. 前日数据为0时，当日非0视为100%变化（触发显著告警），当日也为0视为无变化
• Elastic存储：告警信息包含索引名称、两日数据、变化率、告警类型和时间戳，支持后续查询、可视化和分析

内容的提问来源于stack exchange，提问作者Adewale Ayeni