Golang程序如何清理Prometheus孤儿指标（解决旧值持续上报问题）

核心原因 #

你遇到的问题源于Prometheus Go客户端（github.com/prometheus/client_golang/prometheus）的默认行为：一旦Gauge/Counter等指标被注册，除非显式调用删除方法，否则会永久保留最后一次设置的数值，不会自动清理。

可行解决方案 #

1. 全量同步原数据源，清理本地冗余指标 #

每次从原Prometheus拉取完整指标集合后，对比本地维护的指标标签集，删除本地存在但原数据源已消失的条目：

import (
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/common/model"
)

// 用GaugeVec管理带自定义标签的指标
var memoryAllocatedGauge = prometheus.NewGaugeVec(
    prometheus.GaugeOpts{
        Name: "new_billing_raw:memory_allocated:sum",
        Help: "Memory allocated sum with injected labels",
    },
    []string{"billed_namespace", "exported_job", "task", "task_group", "uuaa"},
)

func init() {
    prometheus.MustRegister(memoryAllocatedGauge)
}

func syncMetricsFromPrometheus() error {
    // 1. 从原Prometheus拉取最新指标数据（此处省略拉取逻辑，假设返回model.Vector类型结果）
    latestMetrics, err := fetchLatestMetrics()
    if err != nil {
        return err
    }

    // 2. 收集本地当前所有存在的指标标签集
    currentLabels := make(map[prometheus.Labels]struct{})
    gatherer := prometheus.DefaultGatherer
    metrics, err := gatherer.Gather()
    if err != nil {
        return err
    }
    for _, mf := range metrics {
        if mf.GetName() != "new_billing_raw:memory_allocated:sum" {
            continue
        }
        for _, m := range mf.GetMetric() {
            labels := make(prometheus.Labels)
            for _, lp := range m.GetLabel() {
                labels[lp.GetName()] = lp.GetValue()
            }
            currentLabels[labels] = struct{}{}
        }
    }

    // 3. 同步最新指标并收集新标签集
    latestLabels := make(map[prometheus.Labels]struct{})
    for _, sample := range latestMetrics {
        if string(sample.Metric[model.MetricNameLabel]) != "new_billing_raw:memory_allocated:sum" {
            continue
        }
        labels := prometheus.Labels{}
        for k, v := range sample.Metric {
            labels[string(k)] = string(v)
        }
        latestLabels[labels] = struct{}{}
        memoryAllocatedGauge.With(labels).Set(float64(sample.Value))
    }

    // 4. 删除本地冗余指标
    for labels := range currentLabels {
        if _, exists := latestLabels[labels]; !exists {
            memoryAllocatedGauge.Delete(labels)
        }
    }

    return nil
}

2. 跟踪指标更新时间，定时清理过期条目 #

维护一个字典记录每个标签集的最后更新时间，定期检查并删除超过阈值未更新的指标：

import (
    "sync"
    "time"

    "github.com/prometheus/client_golang/prometheus"
)

var (
    memoryAllocatedGauge = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "new_billing_raw:memory_allocated:sum",
            Help: "Memory allocated sum with injected labels",
        },
        []string{"billed_namespace", "exported_job", "task", "task_group", "uuaa"},
    )
    lastUpdate = make(map[prometheus.Labels]time.Time)
    mu         sync.RWMutex
)

func init() {
    prometheus.MustRegister(memoryAllocatedGauge)
    // 启动每分钟一次的清理任务
    go func() {
        ticker := time.NewTicker(1 * time.Minute)
        defer ticker.Stop()
        for range ticker.C {
            cleanExpiredMetrics()
        }
    }()
}

func updateMetric(labels prometheus.Labels, value float64) {
    mu.Lock()
    defer mu.Unlock()
    memoryAllocatedGauge.With(labels).Set(value)
    lastUpdate[labels] = time.Now()
}

func cleanExpiredMetrics() {
    mu.Lock()
    defer mu.Unlock()
    // 设置过期阈值：超过2分钟未更新则清理
    expireTime := time.Now().Add(-2 * time.Minute)
    for labels, t := range lastUpdate {
        if t.Before(expireTime) {
            memoryAllocatedGauge.Delete(labels)
            delete(lastUpdate, labels)
        }
    }
}

注意事项 #

• 调用Delete方法时，标签集必须与注册时的标签完全匹配，否则无法删除目标指标。
• 过期阈值需结合原exporter的抓取间隔设置，避免误删正常波动的指标。
• 多goroutine更新指标时必须加锁，防止并发修改导致的数据不一致。

内容的提问来源于stack exchange，提问作者GBT55