Grafana Loki租户留存覆盖规则配置失效求助

在Kubernetes开发环境中通过Helm（v6.24.0）以全分布式模式部署Grafana Loki，全局留存时间30h正常生效，但配置的租户留存覆盖规则始终不生效，已按官方文档要求完成配置。

已执行的排查操作 #

• 反复确认租户名称拼写无误
• 验证留存覆盖配置文件路径正确且可访问
• 在compactor Pod中执行命令/usr/bin/loki -config.file=/etc/loki/config/config.yaml -print-config-stderr ，确认主配置已包含per_tenant_override_config: /etc/loki/config/override/loki-tenant-override-rules.yaml
• 检查所有组件日志未发现相关错误信息
• 开启compactor Pod的调试日志

使用的配置（敏感信息已省略） #

Helm Values配置 #

loki:
  podAnnotations:
    kyverno.io/inject-cacerts: enabled
  compactor:
    retention_enabled: true
    delete_request_store: s3
  limits_config:
    retention_period: 30h
    per_tenant_override_config: /etc/loki/config/override/loki-tenant-override-rules.yaml
    max_streams_per_user: 5000000
  schemaConfig:
    configs:
      - from: 2024-04-01
        store: tsdb
        object_store: s3
        schema: v13
        index:
          prefix: loki_index_
          period: 24h
  ingester:
    chunk_encoding: snappy
  commonConfig:
    ring:
      kvstore:
        store: memberlist
  tracing:
    enabled: false
  annotations: 
    kyverno.io/inject-cacerts: enabled
  querier:
    max_concurrent: 4
  storage:
    type: s3
    s3:
      endpoint: "${LOKI_S3_ENDPOINT}"
      accessKeyId: "${LOKI_S3_ACCESS_KEY_ID}"
      secretAccessKey: "${LOKI_S3_SECRET_ACCESS_KEY}"
      s3ForcePathStyle: false
    bucketNames:
      chunks: loki-chunk-dev3
      ruler: loki-ruler-dev
      admin: loki-admin-dev
  podSecurityContext:
    fsGroup: 101
    runAsGroup: 101
    runAsNonRoot: true
    runAsUser: 101
    seccompProfile:
      type: RuntimeDefault
  containerSecurityContext:
    capabilities:
      drop:
        - ALL
    allowPrivilegeEscalation: false
  auth_enabled: true
  analytics:
    reporting_enabled: false
    usage_stats_url: ""


usage_stats:
  enabled: false

gateway:
  replicas: 2
  basicAuth:
    enabled: true
    existingSecret: basic-auth
  podSecurityContext:
    fsGroup: 101
    runAsGroup: 101
    runAsNonRoot: true
    runAsUser: 101
    seccompProfile:
      type: RuntimeDefault
  containerSecurityContext:
    capabilities:
      drop:
        - ALL
    allowPrivilegeEscalation: false

deploymentMode: Distributed



ingester:
  replicas: 6
  extraArgs:
    - "-config.expand-env=true"
  extraEnv:
    - name: LOKI_S3_ENDPOINT
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ENDPOINT
    - name: LOKI_S3_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ACCESS_KEY_ID
    - name: LOKI_S3_SECRET_ACCESS_KEY
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_SECRET_ACCESS_KEY

querier:
  replicas: 3
  maxUnavailable: 2
  extraArgs:
    - "-config.expand-env=true"
  extraEnv:
    - name: LOKI_S3_ENDPOINT
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ENDPOINT
    - name: LOKI_S3_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ACCESS_KEY_ID
    - name: LOKI_S3_SECRET_ACCESS_KEY
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_SECRET_ACCESS_KEY

queryFrontend:
  replicas: 2
  maxUnavailable: 1

queryScheduler:
  replicas: 2

distributor:
  replicas: 3
  maxUnavailable: 2


compactor:
  replicas: 1
  persistence:
    enabled: true
    size: 50Gi
  extraArgs:
    - "-config.expand-env=true"
    - "-log.level=debug"
  extraEnv:
    - name: LOKI_S3_ENDPOINT
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ENDPOINT
    - name: LOKI_S3_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ACCESS_KEY_ID
    - name: LOKI_S3_SECRET_ACCESS_KEY
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_SECRET_ACCESS_KEY
  extraVolumes:
    - name: override-rules
      configMap:
        name: loki-tenant-override-rules
  extraVolumeMounts:
    - name: override-rules
      mountPath: /etc/loki/config/override/

indexGateway:
  replicas: 2
  maxUnavailable: 1
  extraArgs:
    - "-config.expand-env=true"
  extraEnv:
    - name: LOKI_S3_ENDPOINT
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ENDPOINT
    - name: LOKI_S3_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_ACCESS_KEY_ID
    - name: LOKI_S3_SECRET_ACCESS_KEY
      valueFrom:
        secretKeyRef:
          name: loki-block-storage
          key: LOKI_S3_SECRET_ACCESS_KEY

chunksCache:
  enabled: true
  allocatedMemory: 3012
resultsCache:
  enabled: true
  allocatedMemory: 1012


# optional experimental components
bloomPlanner:
  replicas: 0
bloomBuilder:
  replicas: 0
bloomGateway:
  replicas: 0



lokiCanary:
  enabled: false
test:
  enabled: false

# optional experimental components
bloomPlanner:
  replicas: 0
bloomBuilder:
  replicas: 0
bloomGateway:
  replicas: 0


# Zero out replica counts of other deployment modes
backend:
  replicas: 0
read:
  replicas: 0
write:
  replicas: 0
singleBinary:
  replicas: 0
minio:
  enabled: false

memcached:
  podSecurityContext:
    fsGroup: 101
    runAsGroup: 101
    runAsNonRoot: true
    runAsUser: 101
    seccompProfile:
      type: RuntimeDefault
  containerSecurityContext:
    capabilities:
      drop: [ALL]
    allowPrivilegeEscalation: false
memcachedExporter:
  enabled: true
  resources:
    requests: {}
    limits: {}
  containerSecurityContext:
    capabilities:
      drop: [ALL]
    allowPrivilegeEscalation: false

sidecar:
  securityContext:
    capabilities:
      drop:
        - ALL
    allowPrivilegeEscalation: false
rbac:
  pspEnabled: false
  sccEnabled: false
  pspAnnotations: 
  namespaced: true

extraObjects:
- apiVersion: v1
  kind: ConfigMap
  metadata:
    name: loki-tenant-override-rules
  data:
    loki-tenant-override-rules.yaml: |-
      overrides:
        "team_x_logs":
          retention_period: 24h

内容的提问来源于stack exchange，提问作者Muffe