Go HTTP服务Otel链路追踪客户端与服务端链路未关联问题排查

我正在排查Go HTTP服务的OpenTelemetry（Otel）链路追踪问题：客户端与服务端的trace_id完全匹配，但两条链路始终无法关联，服务端链路显示缺失根span。

客户端链路情况：

服务端链路情况：

明明trace_id匹配，为何会出现缺失根span的情况？

客户端代码（已注入追踪信息） #

func triggerPlaywrightTests(ctx context.Context) (err error) {
    subCtx, subSpan := SpanCreator(ctx, "", "triggerPlaywrightTests")
    defer subSpan.End()
    defer subCtx.Done()

    rawhost, _ := os.Hostname()
    hostSplits := strings.Split(rawhost, ".")
    host := strings.ToLower(hostSplits[0])
    ip := GetLocalIP()

    env, err := ValueFromContextKey(subCtx, "environment")
    cust, err := ValueFromContextKey(subCtx, "customer")
    if err != nil {
        return err
    }

    if host == "" || env == "" || cust == "" || ip == "" {
        err = fmt.Errorf("either host, env, cust, or ip are not found in passed ctx")
        return err
    }

    requestData := TriggerPlaywrightTestsRequest{
        Env:      env,
        Customer: cust,
        Hostname: host,
        IP:       ip,
    }

    jsonData, err := json.Marshal(requestData)
    if err != nil {
        return err
    }

    reqCtx, cancel := context.WithTimeout(subCtx, 30*time.Second)
    defer cancel()

    client := &http.Client{
        Transport: otelhttp.NewTransport(http.DefaultTransport),
    }

    req, err := http.NewRequestWithContext(reqCtx, "POST", triggerPlaywrightTestsUrl, bytes.NewBuffer(jsonData))
    if err != nil {
        return err
    }

    req.Header.Set("Content-Type", "application/json")

    resp, err := client.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    responseBody, err := io.ReadAll(resp.Body)
    if err != nil {
        return err
    }

    return err
}

服务端主程序代码（Azure Function HTTP触发器） #

已确认服务端能收到traceparent header，但仍提示缺失根span。

func main(){
...

    tp, err := createStdoutAndHTTPOTELExporter(ctx, azureFunctionsHoneycombEnvAPIKey)
    if err != nil {
        log.Fatalf("failed to create otel exporters %v", err)
    }

    middleware := otelhttp.NewMiddleware("handleRequest1")
    wrappedHandler := middleware(http.HandlerFunc(handleRequest))

    http.Handle("/api/my-function-url", wrappedHandler)

    listenAddr := ":8080"
    if val, ok := os.LookupEnv("FUNCTIONS_CUSTOMHANDLER_PORT"); ok {
        listenAddr = ":" + val
    }

    log.Fatal(http.ListenAndServe(listenAddr, nil))
}

追踪提供者代码 #

func createStdoutAndHTTPOTELExporter(ctx context.Context, azureFunctionsHoneycombEnvAPIKey string) (*sdktrace.TracerProvider, error) {

    var err error
    resource := createOTELResource(Version, BuildTime, VCSTag)
    
    ...

    httpExporter, err := otlphttp.New(
        ctx,
        otlphttp.WithEndpoint("honeycomb-refinery-url:443"),
        otlphttp.WithHeaders(map[string]string{
            "x-honeycomb-team": azureFunctionsHoneycombEnvAPIKey,
        }),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to create http exporter: %v", err)
    }

    var tracerProviderOptions []sdktrace.TracerProviderOption

    tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithBatcher(stdoutExporter), sdktrace.WithBatcher(httpExporter))

    spanLimits := sdktrace.NewSpanLimits()
    spanLimits.EventCountLimit = 1000

    tracerProviderOptions = append(tracerProviderOptions,
        sdktrace.WithRawSpanLimits(spanLimits),
        sdktrace.WithResource(resource))
    tp = sdktrace.NewTracerProvider(tracerProviderOptions...)

    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(
        propagation.NewCompositeTextMapPropagator(
            propagation.TraceContext{},
            propagation.Baggage{},
        ),
    )
    return tp, err
}

请求处理器代码 #

func handleRequest(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    ctx, span := otel.Tracer("").Start(ctx, "handleRequest")
    defer span.End()

    traceparent := r.Header.Get("traceparent")
    span.AddEvent("inbound header traceparent: " + traceparent)

    fmt.Fprintln(w, "r.Method:", r.Method)
    span.AddEvent("request method: " + r.Method)

    if r.Method != "POST" && r.Method != "GET" {
        http.Error(w, "invalid request method. Not handling request", http.StatusMethodNotAllowed)
        err := fmt.Errorf("invalid request method (not POST or GET): %s", r.Method)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }

    defer r.Body.Close()
    var data InboundRequestData
    err := json.NewDecoder(r.Body).Decode(&data)
    if err != nil {
        http.Error(w, err.Error(), http.StatusBadRequest)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }

    span.SetAttributes(attribute.String("http.inboundrequest.ip", data.IP))
    span.SetAttributes(attribute.String("http.inboundrequest.hostname", data.Hostname))

    if data.Customer == "" || data.Env == "" || data.IP == "" || data.Hostname == "" {
        http.Error(w, "ip, hostname, customer, and env must be set in request body", http.StatusBadRequest)
        err := fmt.Errorf("ip, hostname, customer, and env must be set in request body")
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }

    ctx, jwt, err := generateJWT(ctx, AzureKeyVaultName, GitHubAppPrivateKeyAzureKeyName, GitHubAppID)
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }

    ctx, accessTokenURL, err := getAccessTokenURL(ctx, jwt)
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }
    ctx, installationToken, err := getToken(ctx, jwt, accessTokenURL)
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }
    ctx, err = triggerWorkflow(ctx, installationToken, data)
    if err != nil {
        http.Error(w, err.Error(), http.StatusInternalServerError)
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return
    }
}

已尝试的排查方案 #

• 使用otelhttp.NewHandler替代otelhttp.NewMiddleware
• 手动提取traceparent header创建SpanContext
• 确认服务端能正常接收traceparent header

当前使用Honeycomb作为链路分析工具，以上方案均未解决问题。

内容的提问来源于stack exchange，提问作者SuperSecretAndNotSafeFromWork