GCP部署Flink Kubernetes Operator遇Terraform REST客户端配置错误

报错信息 #

╷
│ Error: Failed to construct REST client
│ 
│   with kubernetes_manifest.flink_cluster,
│   on main.tf line 95, in resource "kubernetes_manifest" "flink_cluster":
│   95: resource "kubernetes_manifest" "flink_cluster" {
│ 
│ cannot create REST client: no client config

出错的资源配置代码 #

# Create Flink Session Cluster using CRD
resource "kubernetes_manifest" "flink_cluster" {
  manifest = {
    apiVersion = "flinkoperator.k8s.io/v1beta1"
    kind       = "FlinkDeployment"
    metadata = {
      name      = "flink-session-cluster"
      namespace = kubernetes_namespace.staging.metadata[0].name
    }
    spec = {
      image = "flink:latest"
      flinkVersion = "1.20.0"
      flinkConfiguration = {
        "taskmanager.numberOfTaskSlots" = "2"
        "parallelism.default" = "2"
      }
      serviceAccount = "flink"
      jobManager = {
        replicas = 1
        resource = {
          memory = "2048m"
          cpu    = 1
        }
      }
      taskManager = {
        resource = {
          memory = "2048m"
          cpu    = 1
        }
      }
      podTemplate = {
        spec = {
          containers = [{
            name = "flink-main-container"
            env = [{
              name  = "DISABLE_JEMALLOC"
              value = "true"
            }]
          }]
        }
      }
    }
  }

  depends_on = [
    helm_release.flink_operator,
    kubernetes_service_account.flink,
    kubernetes_cluster_role_binding.flink
  ]
}

完整部署代码 #

# gcloud auth application-default login
provider "kubernetes" {
  host  = "https://${google_container_cluster.default.endpoint}"
  token = data.google_client_config.current.access_token
  client_certificate = base64decode(
    google_container_cluster.default.master_auth[0].client_certificate,
  )
  client_key = base64decode(google_container_cluster.default.master_auth[0].client_key)
  cluster_ca_certificate = base64decode(
    google_container_cluster.default.master_auth[0].cluster_ca_certificate,
  )
}

provider "google" {
  project = var.project_id
  region  = var.region
}

resource "kubernetes_namespace" "staging" {
  metadata {
    name = "staging"
  }
}

resource "google_compute_address" "default" {
  name   = var.network_name
  region = var.region
}

data "google_client_config" "current" {
}

data "google_container_engine_versions" "default" {
  location = var.location
}

resource "google_container_cluster" "default" {
  name               = var.network_name
  location           = var.location
  remove_default_node_pool = true
  initial_node_count       = 1
  min_master_version = data.google_container_engine_versions.default.latest_master_version
  network            = google_compute_subnetwork.default.name
  subnetwork         = google_compute_subnetwork.default.name

  node_config {
    machine_type = "c2-standard-4"
  }

  provisioner "local-exec" {
    when    = destroy
    command = "sleep 90"
  }
}

resource "google_container_node_pool" "primary_nodes" {
  name       = "${var.network_name}-node-pool"
  location   = var.location
  cluster    = google_container_cluster.default.name
  
  initial_node_count = 1

  autoscaling {
    min_node_count = 1
    max_node_count = 5
  }

  node_config {
    machine_type = "c2-standard-4"
    
    oauth_scopes = [
      "https://www.googleapis.com/auth/logging.write",
      "https://www.googleapis.com/auth/monitoring",
      "https://www.googleapis.com/auth/devstorage.read_only"
    ]
  }
}

# Install Flink Kubernetes Operator using Helm
resource "helm_release" "flink_operator" {
  name       = "flink-operator"
  repository = "https://downloads.apache.org/flink/flink-kubernetes-operator-${var.flink_operator_version}/"
  chart      = "flink-kubernetes-operator"
  namespace  = kubernetes_namespace.staging.metadata[0].name

  set {
    name  = "webhook.create"
    value = "true"
  }

  depends_on = [google_container_node_pool.primary_nodes]
}

# Create Flink Session Cluster using CRD
resource "kubernetes_manifest" "flink_cluster" {
  manifest = {
    apiVersion = "flinkoperator.k8s.io/v1beta1"
    kind       = "FlinkDeployment"
    metadata = {
      name      = "flink-session-cluster"
      namespace = kubernetes_namespace.staging.metadata[0].name
    }
    spec = {
      image = "flink:latest"
      flinkVersion = "1.16"
      flinkConfiguration = {
        "taskmanager.numberOfTaskSlots" = "2"
        "parallelism.default" = "2"
      }
      serviceAccount = "flink"
      jobManager = {
        replicas = 1
        resource = {
          memory = "2048m"
          cpu    = 1
        }
      }
      taskManager = {
        resource = {
          memory = "2048m"
          cpu    = 1
        }
      }
      podTemplate = {
        spec = {
          containers = [{
            name = "flink-main-container"
            env = [{
              name  = "DISABLE_JEMALLOC"
              value = "true"
            }]
          }]
        }
      }
    }
  }

  depends_on = [
    helm_release.flink_operator,
    kubernetes_service_account.flink,
    kubernetes_cluster_role_binding.flink
  ]
}

# Create service account for Flink
resource "kubernetes_service_account" "flink" {
  metadata {
    name      = "flink"
    namespace = kubernetes_namespace.staging.metadata[0].name
  }
}

resource "kubernetes_cluster_role_binding" "flink" {
  metadata {
    name = "flink-role-binding"
  }
  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = "edit"
  }
  subject {
    kind      = "ServiceAccount"
    name      = "flink"
    namespace = kubernetes_namespace.staging.metadata[0].name
  }
}

resource "kubernetes_resource_quota" "staging" {
  metadata {
    name      = "staging-quota"
    namespace = kubernetes_namespace.staging.metadata[0].name
  }

  spec {
    hard = {
      "requests.cpu"    = "4"
      "requests.memory" = "16Gi"
      "limits.cpu"      = "20"
      "limits.memory"   = "128Gi"
    }
  }
}

output "load-balancer-ip" {
  value = google_compute_address.default.address
}

解决方案 #

1. 给Kubernetes Provider添加依赖，确保集群就绪 #

Terraform可能在GKE集群和节点池完全就绪前就初始化Kubernetes Provider，导致配置缺失。修改Provider配置，添加depends_on等待节点池创建完成：

provider "kubernetes" {
  host  = "https://${google_container_cluster.default.endpoint}"
  token = data.google_client_config.current.access_token
  client_certificate = base64decode(
    google_container_cluster.default.master_auth[0].client_certificate,
  )
  client_key = base64decode(google_container_cluster.default.master_auth[0].client_key)
  cluster_ca_certificate = base64decode(
    google_container_cluster.default.master_auth[0].cluster_ca_certificate,
  )

  # 等待节点池就绪后再初始化Provider
  depends_on = [google_container_node_pool.primary_nodes]
}

2. 等待Flink CRD完全注册 #

Helm release完成不代表CRD已被K8s API服务器完全识别，添加数据源等待CRD就绪：

# 等待FlinkDeployment CRD注册完成
data "kubernetes_crd" "flink_deployment" {
  metadata {
    name = "flinkdeployments.flinkoperator.k8s.io"
  }

  depends_on = [helm_release.flink_operator]
}

# 更新flink_cluster的依赖，确保CRD就绪后再创建资源
resource "kubernetes_manifest" "flink_cluster" {
  # ... 现有配置 ...
  depends_on = [
    data.kubernetes_crd.flink_deployment,
    kubernetes_service_account.flink,
    kubernetes_cluster_role_binding.flink
  ]
}

3. 使用动态令牌获取方式避免过期 #

静态引用data.google_client_config.current.access_token可能在执行过程中过期，改用exec方式通过gcloud动态获取集群凭据：

provider "kubernetes" {
  host  = "https://${google_container_cluster.default.endpoint}"
  cluster_ca_certificate = base64decode(
    google_container_cluster.default.master_auth[0].cluster_ca_certificate,
  )
  # 通过gcloud动态获取凭据
  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "gcloud"
    args = [
      "container",
      "clusters",
      "get-credentials",
      google_container_cluster.default.name,
      "--location",
      var.location,
      "--project",
      var.project_id,
    ]
  }

  depends_on = [google_container_node_pool.primary_nodes]
}

4. 检查资源执行顺序 #

确保所有Kubernetes资源（如命名空间、ServiceAccount）都间接依赖GKE集群就绪，避免Terraform提前尝试访问未就绪的K8s API。

内容的提问来源于stack exchange，提问作者Baiqing