Terraform模块化部署K8s镜像需两次apply的问题及解决诉求

我是Terraform和Kubernetes新手，写了两个模块：一个用来创建Linode LKE集群并输出kubeconfig内容，另一个用来部署镜像。现在遇到的问题是，第一次执行terraform apply时，因为kubeconfig的本地文件还没生成，Kubernetes和Helm Provider连接失败，必须执行两次才能成功。我尝试了代码里注释掉的配置方式，也报错了，求解决办法。

代码文件 #

main.tf #

module "deploy_lke" {
      source             = "./modules/linode/kubernetes"
      token              = var.token
      k8s_version        = var.k8s_version
      label              = var.label
      region             = var.region
      tags               = var.tags
      instance_type      = var.instance_type
      number_of_instance = var.number_of_instance
      min                = var.min
      max                = var.max
    }
    
    module "deploy_image" {
      source           = "./modules/kubernetes"
      kube_config_path = module.deploy_lke.kubeconfig
      dockerconfigjson = file("./secret/docker-sec.json")
      deploy_name      = var.deploy_name
      desire_replicas  = var.desire_replicas
      image_link       = var.image_link
      image_name       = var.image_name
      image_port       = var.image_port
      ip_type          = var.ip_type
      max_replicas_val = var.max_replicas_val
      min_replicas_val = var.min_replicas_val
      service_name     = var.service_name
    }

./modules/linode/kubernetes/main.tf #

terraform {
  required_providers {
    linode = {
      source  = "linode/linode"
      version = "1.29.4"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.0.0"
    }
  }
}

provider "linode" {
  token = var.token
}

resource "linode_lke_cluster" "gaintplay-web-lke" {
  k8s_version = var.k8s_version
  label       = var.label
  region      = var.region
  tags        = var.tags


  pool {
    type  = var.instance_type
    count = var.number_of_instance
    autoscaler {
      min = var.min
      max = var.max
    }
  }

  lifecycle {
    ignore_changes = [
      pool.0.count
    ]
  }
}

output "kubeconfig" {
  value     = linode_lke_cluster.gaintplay-web-lke.kubeconfig
}


output "api_endpoints" {
  value = linode_lke_cluster.gaintplay-web-lke.api_endpoints
}

./modules/kubernetes/main.tf #

terraform {
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.0.0"
    }
  }
}
resource "local_file" "kube_config_file" {
    content  = var.kube_config_path
    filename = "${path.module}/config"
}
provider "kubernetes" {
  config_path = var.kube_config_path
}
provider "helm" {
  kubernetes {
    config_path = var.kube_config_path
  }
}

resource "kubernetes_secret" "docker_secret" {
  metadata {
    name = "docker-cfg"
  }

  data = {
    ".dockerconfigjson" = var.dockerconfigjson
  }

  type = "kubernetes.io/dockerconfigjson"
}


resource "kubernetes_deployment" "beta" {
  depends_on = [
    kubernetes_secret.docker_secret
  ]
  metadata {
    name      = var.deploy_name
    namespace = "default"
  }
  spec {
    replicas = var.desire_replicas
    selector {
      match_labels = {
        app = var.deploy_name
      }
    }
    template {
      metadata {
        labels = {
          app = var.deploy_name
        }
      }
      spec {
        image_pull_secrets {
          name = kubernetes_secret.docker_secret.metadata[0].name
        }
        container {
          image_pull_policy = "Always"
          image             =  var.image_link
          name              = var.image_name
          port {
            container_port = var.image_port
          }
        }
      }
    }
  }
}
# provider "kubernetes" {
#     host = "${yamldecode(var.kube_config_path).clusters.0.cluster.server}"
#     client_certificate = "${base64decode(yamldecode(var.kube_config_path).users.0.user.client-certificate-data)}"
#     client_key = "${base64decode(yamldecode(var.kube_config_path).users.0.user.client-key-data)}"
#     cluster_ca_certificate = "${base64decode(yamldecode(var.kube_config_path).clusters.0.cluster.certificate-authority-data)}"
# }
# provider "helm" {
#   kubernetes {
#    host = "${yamldecode(var.kube_config_path).clusters.0.cluster.server}"
#     client_certificate = "${base64decode(yamldecode(var.kube_config_path).users.0.user.client-certificate-data)}"
#     client_key = "${base64decode(yamldecode(var.kube_config_path).users.0.user.client-key-data)}"
#     cluster_ca_certificate = "${base64decode(yamldecode(var.kube_config_path).clusters.0.cluster.certificate-authority-data)}"
#   }
# }

错误信息 #

1. 使用默认配置时的错误 #

Invalid attribute in provider configuration

  with module.deploy_image.provider["registry.terraform.io/hashicorp/kubernetes"],
  on modules/kubernetes/main.tf line 13, in provider "kubernetes":
  13:     provider "kubernetes" {

'config_path' refers to an invalid path: "modules/kubernetes/config": stat modules/kubernetes/config: no such file or directory

2. 使用注释配置时的错误 #

│ Error: Unsupported attribute
│ 
│   on main.tf line 35, in provider "kubernetes":
│   35:     host = "${yamldecode(linode_lke_cluster.gaintplay-web-lke.kubeconfig).clusters.0.cluster.server}"
│ 
│ Can't access attributes on a primitive-typed value (string).
╵
╷
│ Error: Unsupported attribute
│ 
│   on main.tf line 36, in provider "kubernetes":
│   36:     client_certificate = "${base64decode(yamldecode(linode_lke_cluster.gaintplay-web-lke.kubeconfig).users.0.user.client-certificate-data)}"
│ 
│ Can't access attributes on a primitive-typed value (string).

解决方案 #

问题根源 #

• Terraform Provider初始化早于资源创建，第一次apply时local_file.kube_config_file还未生成，Provider找不到配置文件。
• 注释配置报错是因为yamldecode解析后的kubeconfig是map结构，不能用.0索引数组，且带横杠的key需要用方括号语法访问。

修复步骤 #

1. 直接使用kubeconfig内容配置Provider

不需要生成本地文件，直接将kubeconfig字符串传给Provider的config参数，修改./modules/kubernetes/main.tf的Provider配置：

# 删除local_file资源
# resource "local_file" "kube_config_file" {
#     content  = var.kube_config_path
#     filename = "${path.module}/config"
# }

provider "kubernetes" {
  config = var.kube_config_path
}

provider "helm" {
  kubernetes {
    config = var.kube_config_path
  }
}

2. 显式声明模块依赖

在main.tf的deploy_image模块中添加依赖，确保LKE集群创建完成后再初始化Kubernetes Provider：

module "deploy_image" {
  # ... 保留原有参数
  depends_on = [module.deploy_lke]
}

3. 修复注释配置的语法（可选）

如果一定要分解kubeconfig参数，需修正索引和key访问方式：

provider "kubernetes" {
  host                   = yamldecode(var.kube_config_path).clusters[0].cluster.server
  client_certificate     = base64decode(yamldecode(var.kube_config_path).users[0].user["client-certificate-data"])
  client_key             = base64decode(yamldecode(var.kube_config_path).users[0].user["client-key-data"])
  cluster_ca_certificate = base64decode(yamldecode(var.kube_config_path).clusters[0].cluster["certificate-authority-data"])
}

provider "helm" {
  kubernetes {
    host                   = yamldecode(var.kube_config_path).clusters[0].cluster.server
    client_certificate     = base64decode(yamldecode(var.kube_config_path).users[0].user["client-certificate-data"])
    client_key             = base64decode(yamldecode(var.kube_config_path).users[0].user["client-key-data"])
    cluster_ca_certificate = base64decode(yamldecode(var.kube_config_path).clusters[0].cluster["certificate-authority-data"])
  }
}

验证 #

修改后执行一次terraform apply，即可完成集群创建和镜像部署，无需重复执行。

内容的提问来源于stack exchange，提问作者Sunil Sapkota