Consul Connect Injector无法连接Consul Server问题求助

环境版本信息 #

• K3s版本：v1.25.6+k3s1 (9176e03c)
• Helm版本：version.BuildInfo{Version:"v3.12.1", GitCommit:"f32a527a060157990e2aa86bf45010dfb3cc8b8d", GitTreeState:"clean", GoVersion:"go1.20.4"}
• Terraform版本：v1.5.2
• Consul版本：v1.16
• Docker Compose版本：v2.19.1
• Docker版本：

Client: Docker Engine - Community
 Version:           24.0.4
 API version:       1.43
 Go version:        go1.20.5
 Git commit:        3713ee1
 Built:             Fri Jul  7 14:50:55 2023
 OS/Arch:           linux/amd64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          24.0.4
  API version:      1.43 (minimum version 1.12)
  Go version:       go1.20.5
  Git commit:       4ffc614
  Built:            Fri Jul  7 14:50:55 2023
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.6.21
  GitCommit:        3dce8eb055cbb6872793272b4f20ed16117344f8
 runc:
  Version:          1.1.7
  GitCommit:        v1.1.7-0-g860f061
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

Consul Server部署配置 #

server.hcl配置文件 #

server           = true
bootstrap        = false
bootstrap_expect = 1

node_name  = "dev-consul"
datacenter = "dev-dc"

encrypt                 = "<encrypt-key>"
encrypt_verify_incoming = true
encrypt_verify_outgoing = true

tls {
  defaults {
    ca_file         = "/consul/config/certs/consul-agent-ca.pem"
    cert_file       = "/consul/config/certs/dev-dc-server-consul-0.pem"
    key_file        = "/consul/config/certs/dev-dc-server-consul-0-key.pem"
    verify_incoming = true
    verify_outgoing = true
  }
}

data_dir  = "/consul-data"
log_level = "INFO"

advertise_addr = "<advertise-addr>"
bind_addr      = "0.0.0.0"

addresses = {
  "http" = "0.0.0.0"
}

auto_encrypt = {
  "allow_tls" = true
}

connect = {
  "enabled" = true
}

ui_config = {
  "enabled" = true
}

Docker Compose部署文件 #

version: "3.9"

services:
  consul:
    image: hashicorp/consul:1.16
    volumes:
      - ./config/server.hcl:/consul/config/server.hcl:ro
      - consul_data:/titanium/consul-data
      - ./certs:/consul/config/certs/
    ports:
      - "8600:8600/tcp"
      - "8600:8600/udp"
      - "8500:8500/tcp"
      - "8500:8500/udp"
      - "8301:8301/tcp"
      - "8301:8301/udp"
      - "8302:8302/tcp"
      - "8302:8302/udp"
      - "8502:8502"
      - "21000-21255:21000-21255"
      - "8300:8300"
      - "8300:8300/udp"
    command: "agent"

volumes:
  consul_data:

Helm Chart部署Consul Client配置 #

初始values.yml #

global:
  name: consul
  image: hashicorp/consul:1.16
  domain: dev.local
  datacenter: dev-dc
  exposeGossipPorts: true
  gossipEncryption:
    secretName: "gossip-encryption-key-secret"
    secretKey: "key"
  tls:
    enabled: true
    enableAutoEncrypt: true
    verify: true
    caCert:
      secretName: "consul-certs"
      secretKey: "ca.pem"
connectInject:
  enabled: true
  default: true
  cni:
    enabled: true
    logLevel: info
    cniBinDir: "/opt/cni/bin"
    cniNetDir: "/etc/cni/net.d"
  namespaceSelector: |
    matchLabels:
      connect-inject : enabled
  failurePolicy: "Ignore"
server:
  enabled: false
client:
  enabled: true
  join: [ "<consul-service-addr>" ]
  grpc: true

简化后的values.yml #

global:
  name: consul
  domain: dev.local
  datacenter: dev-dc
  gossipEncryption:
    secretName: "gossip-encryption-key-secret"
    secretKey: "key"
  tls:
    enabled: true
    enableAutoEncrypt: true
    verify: true
    caCert:
      secretName: "consul-certs"
      secretKey: "ca.pem"
connectInject:
  enabled: true
  failurePolicy: "Ignore"
controller:
  enabled: true
server:
  enabled: false
client:
  enabled: true
  image: hashicorp/consul:1.15.3
  join: [ "<consul-server-addr>" ]

Terraform部署脚本 #

provider "helm" {
  kubernetes {
    config_path = "~/.kube/config"
  }
}

resource "helm_release" "consul-connect" {
  name       = "consul-connect"
  chart      = "consul"
  repository = "https://helm.releases.hashicorp.com"
  namespace  = "consul"

  values = [file("config/values.yml")]
}

问题现象 #

Consul Client可正常连接集群，但Connect Injector Pod无法启动，出现以下报错：

Pod日志错误 #

2023-07-23T19:23:44.851Z [INFO]  consul-server-connection-manager: trying to connect to a Consul server 2023-07-23T19:23:44.854Z [ERROR] consul-server-connection-manager: connection error: error="failed to discover Consul server addresses: failed to resolve DNS name: consul-server.consul.svc: lookup consul-server.consul.svc on 10.43.0.10:53: no such host

Kubernetes事件错误 #

MountVolume.SetUp failed for volume "consul-ca-cert"
MountVolume.SetUp failed for volume "certs" : secret "consul-connect-inject-webhook-cert" not found

已尝试的动作 #

• 多次修改values.yml配置，包括简化配置内容
• 尝试通过Helm Chart部署Consul Server，仍无法解决Connect Injector Pod启动问题
• 已手动创建gossip加密密钥和TLS证书对应的Kubernetes Secret

需要协助排查该问题。

内容的提问来源于stack exchange，提问作者Mathias Dollenbacher