如何在Kubernetes Ingress中为Loki传递动态请求头

云上老周

2026-8-11

问题描述

本地VM上的Nginx配置运行正常，通过两层认证（日志写入、日志读取）向Loki传输日志。现在要把这个Nginx迁移到Kubernetes环境，计划用负载均衡地址projectname.dev.url.com，通过/loki/api/v1/push路径写入日志，/logs路径读取日志。目前已经能用curl成功访问读取路径，但写入路径无法推送日志，原因是缺少所需的请求头。我们已经配置了Ingress和包含map规则的ConfigMap，核心疑问是：如何将$remote_user映射至basic-auth Secret中的用户名，实现和VM环境一致的请求头传递效果。

本地VM的Nginx配置

upstream "FQDN Hostname2" {
# server XXXXXXXXXXXXXXXXX:443 weight=1;
  server XXXXXXXXXXXXXXXXX:443 weight=1;
  server XXXXXXXXXXXXXXXXX:443 weight=1;
  server XXXXXXXXXXXXXXXXX:443 weight=1;
  server XXXXXXXXXXXXXXXXX:443 weight=1;
}
map $remote_user $loki_scope_org_id {
  "~^(?<scope_org_id>[^#]*)"  $scope_org_id;
}

server {
    listen 443 ssl;
    server_name FQDN HOstname;
    ssl_certificate /etc/pki/tls/certs/FQDN HOstname.crt;
    ssl_certificate_key /etc/pki/tls/private/FQDN HOstname.key;
    ssl_protocols TLSv1.2 TLSv1.3;
    client_max_body_size 2G;
    client_body_buffer_size 50M;
    location /loki/api/v1/push {
        log_subrequest on;
        mirror /loki/api/v2/push;
        auth_basic "Please provide X-Scope-OrgID";
        auth_basic_user_file /etc/nginx/.htpasswd_agents; # 写入认证凭证
        proxy_pass http://FQDN HOstname:3100/loki/api/v1/push;
        proxy_set_header X-Scope-OrgID $loki_scope_org_id;
    }
    location /loki/api/v2/push {
        internal;
        log_subrequest on;
        auth_basic "Please provide X-Scope-OrgID";
        auth_basic_user_file /etc/nginx/.htpasswd_agents;
        proxy_pass https://FQDN Hostname2/logs/loki/api/v1/push;
        proxy_set_header X-Scope-OrgID $loki_scope_org_id;
    }
    location / {
        auth_basic "Please provide X-Scope-OrgID";
        auth_basic_user_file /etc/nginx/.htpasswd_grafana;
        proxy_pass http://FQDN HOstname:3100/;
        proxy_set_header X-Scope-OrgID $remote_user;
    }
    location /logs {
        auth_basic "Please provide X-Scope-OrgID";
        auth_basic_user_file /etc/nginx/.htpasswd_grafana;
        proxy_pass https://FQDN Hostname2/logs;
        proxy_set_header X-Scope-OrgID $remote_user;
    }

    access_log  /var/lib/loki/nginx/LOGNAME_access.log  main;
    error_log /var/lib/loki/nginx/LOGNAME_error.log warn;
}

Kubernetes Ingress配置

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    meta.helm.sh/release-name: loki
    meta.helm.sh/release-namespace: loki
    nginx.ingress.kubernetes.io/auth-realm: Authentication Required
    nginx.ingress.kubernetes.io/auth-secret: basic-auth
    nginx.ingress.kubernetes.io/auth-type: basic
    nginx.ingress.kubernetes.io/rewrite-target: /$2
  creationTimestamp: "2022-10-26T20:34:42Z"
  generation: 9
  labels:
    app.kubernetes.io/component: gateway
    app.kubernetes.io/instance: loki
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: loki-distributed
    app.kubernetes.io/version: 2.4.2
    helm.sh/chart: loki-distributed-0.47.4
  name: loki-loki-distributed-gateway
  namespace: loki
  resourceVersion: "94038120"
  uid: a9175373-458b-41ff-a261-d984e266a54e
spec:
  ingressClassName: nginx
  rules:
  - http:
      paths:
      - backend:
          service:
            name: loki-loki-distributed-gateway
            port:
              number: 443
        path: /logs(/|$)(.*)
        pathType: Prefix
      - backend:
          service:
             name: loki-loki-distributed-distributor
             port:
               number: 443
        path: /logs/loki/api/v1/push
        pathType: Exact
  tls:
  - hosts:
    - '*.fqdn.fqdn.net'
status:
  loadBalancer:
    ingress:
    - ip: X.X.X.159
    - ip: X.X.X.17
    - ip: X.X.X.49
    - ip: X.X.X.72
    - ip: X.X.X.76
    - ip: X.X.X.78
    - ip: X.X.X.83
    - ip: X.X.X.120
    - ip: X.X.X.123
    - ip: X.X.X.127
    - ip: X.X.X.144
    - ip: X.X.X.145
    - ip: X.X.X.167
    - ip: X.X.X.168
    - ip: X.X.X.171
    - ip: X.X.X.177
    - ip: X.X.X.178
    - ip: X.X.X.183
    - ip: X.X.X.185
    - ip: X.X.X.186
    - ip: X.X.X.190
    - ip: X.X.X.197
    - ip: X.X.X.4
    - ip: X.X.X.6

自定义Ingress ConfigMap配置

worker_processes  5;  ## Default: 1
error_log  /dev/stderr;
pid        /tmp/nginx.pid;
worker_rlimit_nofile 8192;

events {
  worker_connections  4096;  ## Default: 1024
}

http {
  client_body_temp_path /tmp/client_temp;
  proxy_temp_path       /tmp/proxy_temp_path;
  fastcgi_temp_path     /tmp/fastcgi_temp;
  uwsgi_temp_path       /tmp/uwsgi_temp;
  scgi_temp_path        /tmp/scgi_temp;
  map $remote_user $loki_scope_org_id {
    "~^(?<scope_org_id>[^#]*)"  $scope_org_id;
  }

  default_type application/octet-stream;
  log_format   main '$remote_addr - $remote_user [$time_local]  $status '
        '"$request" $body_bytes_sent "$http_referer" '
        '"$http_user_agent" "$http_x_forwarded_for"';
  access_log   /dev/stderr  main;
  sendfile     on;
  tcp_nopush   on;
  resolver rke2-coredns-rke2-coredns.kube-system.svc.cluster.local;

  server {
    listen             8080;

    location = / {
      return 200 'OK';
      auth_basic off;
    }

    location = /api/prom/push {
      proxy_pass       http://loki-loki-distributed-distributor.loki.svc.cluster.local:3100/api/prom/push;
    }

    location = /api/prom/tail {
      proxy_pass       http://loki-loki-distributed-querier.loki.svc.cluster.local:3100/api/prom/tail;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
    }

    location ~ /api/prom/.* {
      proxy_pass       http://loki-loki-distributed-query-frontend.loki.svc.cluster.local:3100/api/prom/$1;
    }

    location = /loki/api/v1/push {
      proxy_pass       http://loki-loki-distributed-distributor.loki.svc.cluster.local:3100/loki/api/v1/push;
      proxy_set_header X-Scope-OrgID $loki_scope_org_id;
    }

    location = /loki/api/v1/tail {
      proxy_pass       http://loki-loki-distributed-querier.loki.svc.cluster.local:3100/loki/api/v1/tail;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection "upgrade";
    }

    location ~ /loki/api/.* {
      proxy_pass       http://loki-loki-distributed-query-frontend.loki.svc.cluster.local:3100/loki/api/$1;
    }
  }
}

解决方案

1. 确认`$remote_user`的传递逻辑

Nginx Ingress Controller完成basic-auth认证后，会自动把认证用户名存入$remote_user变量，这个变量可以直接在ConfigMap的配置中使用。你当前ConfigMap里的map $remote_user $loki_scope_org_id规则本身没问题，只要$remote_user能正确获取到用户名，映射就会生效。

2. 拆分两层认证（写入/读取用不同Secret）

VM环境中写入用的是.htpasswd_agents，读取用的是.htpasswd_grafana，所以K8s里要创建两个对应的basic-auth Secret：

basic-auth-agents：对应写入认证的用户名密码
basic-auth-grafana：对应读取认证的用户名密码

创建Secret的命令：

# 创建写入用的Secret
htpasswd -c ./agents.htpasswd 写入用户名
kubectl create secret generic basic-auth-agents --from-file=agents.htpasswd -n loki

# 创建读取用的Secret
htpasswd -c ./grafana.htpasswd 读取用户名
kubectl create secret generic basic-auth-grafana --from-file=grafana.htpasswd -n loki

3. 调整Ingress配置，给不同路径分配对应认证

修改Ingress，给写入路径和读取路径分别配置认证Secret：

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    meta.helm.sh/release-name: loki
    meta.helm.sh/release-namespace: loki
    nginx.ingress.kubernetes.io/rewrite-target: /$2
    # 读取路径的全局认证
    nginx.ingress.kubernetes.io/auth-realm: "Please provide X-Scope-OrgID"
    nginx.ingress.kubernetes.io/auth-secret: basic-auth-grafana
    nginx.ingress.kubernetes.io/auth-type: basic
    # 写入路径单独覆盖认证配置
    nginx.ingress.kubernetes.io/server-snippet: |
      location = /loki/api/v1/push {
        auth_basic "Please provide X-Scope-OrgID";
        auth_basic_user_file /etc/nginx/secrets/basic-auth-agents;
      }
  labels:
    app.kubernetes.io/component: gateway
    app.kubernetes.io/instance: loki
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: loki-distributed
    app.kubernetes.io/version: 2.4.2
    helm.sh/chart: loki-distributed-0.47.4
  name: loki-loki-distributed-gateway
  namespace: loki
spec:
  ingressClassName: nginx
  rules:
  - host: projectname.dev.url.com
    http:
      paths:
      - backend:
          service:
            name: loki-loki-distributed-gateway
            port:
              number: 443
        path: /logs(/|$)(.*)
        pathType: Prefix
      - backend:
          service:
             name: loki-loki-distributed-distributor
             port:
               number: 443
        path: /loki/api/v1/push
        pathType: Exact
  tls:
  - hosts:
    - projectname.dev.url.com
    secretName: your-tls-secret # 替换为你的域名TLS Secret

4. 完善ConfigMap中的请求头传递

在ConfigMap的/loki/api/v1/push和/logs路径中，确保正确设置X-Scope-OrgID请求头：

# 写入路径
location = /loki/api/v1/push {
  proxy_pass       http://loki-loki-distributed-distributor.loki.svc.cluster.local:3100/loki/api/v1/push;
  proxy_set_header X-Scope-OrgID $loki_scope_org_id; # 使用map后的变量
  proxy_set_header Host $host;
}

# 读取路径
location /logs {
  proxy_pass       https://FQDN_Hostname2/logs;
  proxy_set_header X-Scope-OrgID $remote_user; # 直接用$remote_user
  proxy_set_header Host $host;
}

5. 验证配置

用curl测试写入路径，查看请求头是否包含X-Scope-OrgID：

curl -u 写入用户名:写入密码 -H "Content-Type: application/json" -X POST https://projectname.dev.url.com/loki/api/v1/push -d '{"streams":[{"stream":{"foo":"bar"},"values":[["1620000000000","test log"]]}]}' -v

查看输出中的请求头部分，确认X-Scope-OrgID是否正确设置，同时检查Loki是否成功接收日志。

内容的提问来源于stack exchange，提问作者fid740

如何在Kubernetes Ingress中为Loki传递动态请求头

本地VM的Nginx配置 #

Kubernetes Ingress配置 #

自定义Ingress ConfigMap配置 #

1. 确认$remote_user的传递逻辑 #

2. 拆分两层认证（写入/读取用不同Secret） #

3. 调整Ingress配置，给不同路径分配对应认证 #

4. 完善ConfigMap中的请求头传递 #

5. 验证配置 #