Terraform中Kubernetes/Helm Provider读取EKS集群数据报错

使用Terraform的kubernetes和helm Provider时，读取EKS集群模块数据遇到问题。最初尝试用cluster_name配合data资源读取，但集群尚未创建导致失败。改为直接从eks/aws模块读取时，出现以下错误：

│ Error: Unsupported attribute
│ 
│   on providers.tf line 37, in provider "kubernetes":
│   37:   cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data.data)
│     ├────────────────
│     │ module.primary.cluster_certificate_authority_data is "**********************"
│ 
│ Can't access attributes on a primitive-typed value (string).
╵
╷
│ Error: Unsupported attribute
│ 
│   on providers.tf line 54, in provider "helm":
│   54:     cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data.data)
│     ├────────────────
│     │ module.primary.cluster_certificate_authority_data is "**********************"
│ 
│ Can't access attributes on a primitive-typed value (string).

相关配置文件 #

main.tf #

################################################
#          KMS CLUSTER ENCRYPTION KEY          #
################################################
module "kms" {
  source  = "terraform-aws-modules/kms/aws"
  version = "1.1.0"

  aliases               = ["eks/${var.cluster_name}__cluster_encryption_key_test"]
  description           = "${var.cluster_name} cluster encryption key"
  enable_default_policy = true
  key_owners            = [data.aws_caller_identity.current.arn]

  tags = local.tags
}

##################################
#       KUBERNETES CLUSTER       #
##################################
module "primary" {
  source  = "terraform-aws-modules/eks/aws"
  version = "~> 19.13.1"

  cluster_name                    = var.cluster_name
  cluster_version                 = var.cluster_version
  cluster_endpoint_private_access = var.cluster_endpoint_private_access
  cluster_endpoint_public_access  = var.cluster_endpoint_public_access

  create_kms_key = false
  cluster_encryption_config = {
    resources        = ["secrets"]
    provider_key_arn = module.kms.key_arn
  }

  create_cni_ipv6_iam_policy = var.create_cni_ipv6_iam_policy
  manage_aws_auth_configmap  = true
  aws_auth_roles             = var.aws_auth_roles

  vpc_id     = var.vpc_id
  subnet_ids = var.subnet_ids

  eks_managed_node_group_defaults = {
    ami_type       = var.ami_type
    disk_size      = var.disk_size
    instance_types = var.instance_types

    iam_role_attach_cni_policy = var.iam_role_attach_cni_policy
  }

  eks_managed_node_groups = {
    primary = {
      min_size     = 1
      max_size     = 5
      desired_size = 1

      capacity_type = "ON_DEMAND"
    }
    secondary = {
      min_size     = 1
      max_size     = 5
      desired_size = 1

      capacity_type = "SPOT"
    }
  }

  cluster_addons = {
    coredns = {
      most_recent                 = true

      resolve_conflicts_on_create = "OVERWRITE"
      resolve_conflicts_on_update = "PRESERVE"

      timeouts = {
        create = "20m"
        delete = "20m"
        update = "20m"
      }
    }
    kube-proxy = {
      most_recent       = true
      resolve_conflicts_on_create = "OVERWRITE"
      resolve_conflicts_on_update = "PRESERVE"

      timeouts = {
        create = "20m"
        delete = "20m"
        update = "20m"
      }
    }
    aws-ebs-csi-driver = {
      most_recent       = true
      resolve_conflicts_on_create = "OVERWRITE"
      resolve_conflicts_on_update = "PRESERVE"

      timeouts = {
        create = "20m"
        delete = "20m"
        update = "20m"
      }
    }
    vpc-cni = {
      most_recent       = true
      resolve_conflicts_on_create = "OVERWRITE"
      resolve_conflicts_on_update = "PRESERVE"

      timeouts = {
        create = "20m"
        delete = "20m"
        update = "20m"
      }
    }
  }

  fargate_profiles = {
    default = {
      name = "default"
      selectors = [
        {
          namespace = "kube-system"
          labels = {
            k8s-app = "kube-dns"
          }
        },
        {
          namespace = "default"
        }
      ]

      timeouts = {
        create = "20m"
        delete = "20m"
      }
    }
  }

  tags = {
    repo  = "https://github.com/impinj-di/terraform-aws-eks-primary"
    team  = "di"
    owner = "di_admins@impinj.com"
  }
}

####################################
#       KUBERNETES RESOURCES       #
####################################
resource "kubernetes_namespace" "this" {
  depends_on = [module.primary]
  for_each   = toset(local.eks_namespaces)
  metadata {
    name = each.key
  }
}

providers.tf #

terraform {
  required_version = ">= 1.3.7"

  required_providers {
    aws = ">= 4.12.0"
    # harness = {
    #   source = "harness/harness"
    #   version = "0.21.0"
    # }
    helm = {
      source = "hashicorp/helm"
      version = "2.9.0"
    }
    kubernetes = {
      source = "hashicorp/kubernetes"
      version = "2.11.0"
    }
  }
}

terraform {
  backend "s3" {
    bucket  = "impinj-canary-terraform"
    key     = "terraform-aws-eks-primary.tfstate"
    region  = "us-west-2"
    encrypt = true
  }
}

provider "aws" {
  alias  = "sec"
  region = "us-west-2"
}

provider "kubernetes" {
  host                   = module.primary.cluster_endpoint
  cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data.data)
  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "aws"
    args = ["eks", "get-token", "--cluster-name", module.primary.cluster_name]
  }
}

# provider "harness" {
#   endpoint         = "https://app.harness.io/gateway"
#   account_id       = var.harness_account_id
#   platform_api_key = var.harness_platform_api_key
# }

provider "helm" {
  kubernetes {
    host                   = module.primary.cluster_endpoint
    cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data.data)
    exec {
      api_version = "client.authentication.k8s.io/v1beta1"
      command     = "aws"
      args        = ["eks", "get-token", "--cluster-name", module.primary.cluster_name]
    }
  }
}

解决方案 #

错误原因明确：module.primary.cluster_certificate_authority_data本身就是一个base64编码的字符串（不是对象），错误地尝试访问它的.data属性，导致Terraform报错“无法在原始类型值（字符串）上访问属性”。

修改providers.tf中的两处配置，去掉.data后缀即可：

修改后的kubernetes provider配置 #

provider "kubernetes" {
  host                   = module.primary.cluster_endpoint
  cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data)
  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "aws"
    args = ["eks", "get-token", "--cluster-name", module.primary.cluster_name]
  }
}

修改后的helm provider配置 #

provider "helm" {
  kubernetes {
    host                   = module.primary.cluster_endpoint
    cluster_ca_certificate = base64decode(module.primary.cluster_certificate_authority_data)
    exec {
      api_version = "client.authentication.k8s.io/v1beta1"
      command     = "aws"
      args        = ["eks", "get-token", "--cluster-name", module.primary.cluster_name]
    }
  }
}

修改后，Terraform就能正确解码CA证书，不会再出现属性访问错误。

内容的提问来源于stack exchange，提问作者R. Barrett