如何在Jaeger中配置SPM？配置后UI无数据请求排查

我近期尝试搭建SPM监控，但Jaeger UI页面始终无数据显示，所有配置均基于OpenTelemetry Collector Contrib的spanmetricsconnector示例。以下是我的全部配置及代码：

Go代码 #

// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

// Example using OTLP exporters + collector + third-party backends. For
// information about using the exporter, see:
// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp?tab=doc#example-package-Insecure
package main

import (
    "context"
    "fmt"
    "log"
    "os"
    "os/signal"
    "time"

    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/propagation"
    sdkmetric "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
    "go.opentelemetry.io/otel/trace"
)

var serviceName = semconv.ServiceNameKey.String("test-service")

// Initialize a gRPC connection to be used by both the tracer and meter
// providers.
func initConn() (*grpc.ClientConn, error) {
    // It connects the OpenTelemetry Collector through local gRPC connection.
    // You may replace `localhost:4317` with your endpoint.
    conn, err := grpc.NewClient("localhost:4317",
        // Note the use of insecure transport here. TLS is recommended in production.
        grpc.WithTransportCredentials(insecure.NewCredentials()),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create gRPC connection to collector: %w", err)
    }

    return conn, err
}

// Initializes an OTLP exporter, and configures the corresponding trace provider.
func initTracerProvider(ctx context.Context, res *resource.Resource, conn *grpc.ClientConn) (func(context.Context) error, error) {
    // Set up a trace exporter
    traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    // Register the trace exporter with a TracerProvider, using a batch
    // span processor to aggregate spans before export.
    bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
    tracerProvider := sdktrace.NewTracerProvider(
        sdktrace.WithSampler(sdktrace.AlwaysSample()),
        sdktrace.WithResource(res),
        sdktrace.WithSpanProcessor(bsp),
    )
    otel.SetTracerProvider(tracerProvider)

    // Set global propagator to tracecontext (the default is no-op).
    otel.SetTextMapPropagator(propagation.TraceContext{})

    // Shutdown will flush any remaining spans and shut down the exporter.
    return tracerProvider.Shutdown, nil
}

// Initializes an OTLP exporter, and configures the corresponding meter provider.
func initMeterProvider(ctx context.Context, res *resource.Resource, conn *grpc.ClientConn) (func(context.Context) error, error) {
    metricExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create metrics exporter: %w", err)
    }

    meterProvider := sdkmetric.NewMeterProvider(
        sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)),
        sdkmetric.WithResource(res),
    )
    otel.SetMeterProvider(meterProvider)

    return meterProvider.Shutdown, nil
}

func main() {
    log.Printf("Waiting for connection...")

    ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
    defer cancel()

    conn, err := initConn()
    if err != nil {
        log.Fatal(err)
    }

    res, err := resource.New(ctx,
        resource.WithAttributes(
            // The service name used to display traces in backends
            serviceName,
        ),
    )
    if err != nil {
        log.Fatal(err)
    }

    shutdownTracerProvider, err := initTracerProvider(ctx, res, conn)
    if err != nil {
        log.Fatal(err)
    }
    defer func() {
        if err := shutdownTracerProvider(ctx); err != nil {
            log.Fatalf("failed to shutdown TracerProvider: %s", err)
        }
    }()

    shutdownMeterProvider, err := initMeterProvider(ctx, res, conn)
    if err != nil {
        log.Fatal(err)
    }
    defer func() {
        if err := shutdownMeterProvider(ctx); err != nil {
            log.Fatalf("failed to shutdown MeterProvider: %s", err)
        }
    }()

    tracer := otel.Tracer("test-tracer")
    meter := otel.Meter("test-meter")

    // Attributes represent additional key-value descriptors that can be bound
    // to a metric observer or recorder.
    commonAttrs := []attribute.KeyValue{
        attribute.String("attrA", "chocolate"),
        attribute.String("attrB", "raspberry"),
        attribute.String("attrC", "vanilla"),
    }

    runCount, err := meter.Int64Counter("run", metric.WithDescription("The number of times the iteration ran"))
    if err != nil {
        log.Fatal(err)
    }

    // Work begins
    ctx, span := tracer.Start(
        ctx,
        "CollectorExporter-Example",
        trace.WithAttributes(commonAttrs...))
    defer span.End()
    for i := 0; i < 1; i++ {
        _, iSpan := tracer.Start(ctx, fmt.Sprintf("Sample-%d", i))
        runCount.Add(ctx, 1, metric.WithAttributes(commonAttrs...))
        log.Printf("Doing really hard work (%d / 10)\n", i+1)

        <-time.After(time.Second)
        iSpan.End()
    }

    log.Printf("Done!")
}

docker-compose.yml #

# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel-collector.yaml"]
    volumes:
      - ./otel-collector.yaml:/etc/otel-collector.yaml
    ports:
      - 4317:4317
      - 8889:8889

  prometheus:
    image: prom/prometheus:v2.52.0
    volumes:
      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
    ports:
      - 9090:9090

  jaeger:
    image: jaegertracing/all-in-one:latest
    volumes:
      - "./jaeger-ui.json:/etc/jaeger/jaeger-ui.json"
    command: --query.ui-config /etc/jaeger/jaeger-ui.json
    environment:
      - METRICS_STORAGE_TYPE=prometheus
      - PROMETHEUS_SERVER_URL=http://prometheus:9090
      - PROMETHEUS_QUERY_SUPPORT_SPANMETRICS_CONNECTOR=true
      - PROMETHEUS_QUERY_NAMESPACE=testapp

      - PROMETHEUS_QUERY_NORMALIZE_CALLS=true
      - PROMETHEUS_QUERY_NORMALIZE_DURATION=true
    ports:
      - 16686:16686
      - 16687:16687

jaeger-ui.json #

{
  "monitor": {
    "menuEnabled": true
  },
  "dependencies": {
    "menuEnabled": true
  }
}

otel-collector.yaml #

# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
processors:
extensions:
  health_check: {}
exporters:
  otlp:
    endpoint: jaeger:4317
    tls:
      insecure: true
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: testapp
  prometheusremotewrite:
    endpoint: http://localhost:9090/api/v1/write
    target_info:
      enabled: true
    remote_write_queue:
      enabled: True
      queue_size: 100000
      num_consumers: 5
  debug:
connectors:
  spanmetrics:
    namespace: span.metrics
  count:

service:
  extensions: [health_check]
  pipelines:
    traces:
      receivers: [otlp]
      processors: []
      exporters: [otlp, debug, spanmetrics]

    metrics:
      receivers: [spanmetrics, otlp]
      processors: []
      exporters: [prometheus, prometheusremotewrite]

prometheus.yaml #

# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

scrape_configs:
  - job_name: "otel-collector"
    scrape_interval: 5s
    static_configs:
      - targets: ["otel-collector:8889"]

remote_write:
  - url: 'http://localhost:9090/api/v1/write'

排查要点 #

• 链路基础验证

  1. 查看otel-collector日志，确认是否收到trace数据：docker logs <otel-collector容器ID>，搜索Received spans关键词。
  2. 访问Jaeger UI（http://localhost:16686），搜索test-service，先确认基础trace数据是否能正常显示。

• SpanMetrics配置匹配

  1. 当前otel-collector中spanmetrics的namespace为span.metrics，但Jaeger环境变量PROMETHEUS_QUERY_NAMESPACE设置为testapp，二者不匹配，需统一其中一方的配置。
  2. 检查otel-collector日志，确认spanmetrics连接器是否成功加载，搜索spanmetricsconnector关键词。

• Prometheus数据流向

  1. 访问Prometheus UI（http://localhost:9090），搜索testapp_span开头的指标，确认是否生成了span metrics。
  2. 查看Prometheus的Targets页面（http://localhost:9090/targets），确认otel-collector采集目标状态为UP。
  3. 移除prometheus.yaml中的remote_write配置，该配置在当前环境下属于冗余设置，可能引发数据写入异常。

• Jaeger与Prometheus集成

  1. 查看Jaeger日志，确认是否能正常连接到Prometheus：docker logs <jaeger容器ID>，搜索prometheus关键词，排查连接错误。
  2. 确认Jaeger的PROMETHEUS_QUERY_SUPPORT_SPANMETRICS_CONNECTOR=true环境变量已正确设置，这是开启SpanMetrics支持的关键。

• 代码调整

  1. 将Go代码中的循环条件改为i < 10，增加生成的span数量，避免因数据量过少导致采集不及时。

内容的提问来源于stack exchange，提问作者Oscar Chua Wei Wen