Terraform AzureRM/AKS创建AKS集群时podCIDR修改不允许报错

最近使用Terraform AzureRM/AKS模块（版本7.5.0）创建Kubernetes 1.27.9版本的AKS集群时出现故障。由于后续将安装Calico CNI，我们配置了以下网络相关参数：

network_plugin                      = "none" # 后续安装Calico，因此设置为none
net_profile_pod_cidr                = null
net_profile_service_cidr            = null
net_profile_dns_service_ip          = null
net_profile_outbound_type           = "userDefinedRouting"

集群创建完成后，执行后续操作时返回400错误：

│ RESPONSE 400: 400 Bad Request
│ ERROR CODE: PropertyChangeNotAllowed
│ --------------------------------------------------------------------------------
│ {
│   "code": "PropertyChangeNotAllowed",
│   "details": null,
│   "message": "Changing property 'networkProfile.podCIDR' from '10.244.0.0/16' to '' is not allowed.",
│   "subcode": "",
│   "target": "networkProfile.podCIDR"
│ }
│ --------------------------------------------------------------------------------
│ 
│ 
│   with module.workload.module.aks.azapi_update_resource.aks_cluster_post_create,
│   on .terraform/modules/workload.aks/main.tf line 562, in resource "azapi_update_resource" "aks_cluster_post_create":
│  562: resource "azapi_update_resource" "aks_cluster_post_create" {

可见即使我们显式将net_profile_pod_cidr设为null，该参数仍被设置为默认值10.244.0.0/16。我们尝试了以下几种pod_cidr配置方式，但均失败：

• 设置为空字符串
• 不设置该参数
• 设置为"none"
• 设置为默认值10.244.0.0/16

该问题自我们使用的Azure区域发布AKS新版本后出现。以下是可复现问题的简化代码示例：

locals {
  cluster_subnet_id                     = "YOURSUBNETID"
  aks_kubernetes_version                = "1.27.9"
  rbac_aad_admin_group_object_ids       = [ "YOUR AAD ADMIN GROUP" ]
  rg_name                               = "YOUR RESOURCE GROUP"
  location                              = "australiaeast"
  aks = {
    agents_availability_zones           = [1,2,3]
    agents_count                        = null # 启用自动扩缩容时设为null
    enable_auto_scaling                 = true
    enable_host_encryption              = false
    identity_type                       = "SystemAssigned"
    ingress_application_gateway_enabled = false
    key_vault_secrets_provider_enabled  = true
    kms_enabled                         = false
    kms_key_vault_network_access        = "Private"
    log_analytics_workspace_enabled     = false
    network_plugin                      = "none" # 后续安装Calico，因此设置为none
    net_profile_pod_cidr                = null
    net_profile_service_cidr            = null
    net_profile_dns_service_ip          = null
    net_profile_outbound_type           = "userDefinedRouting"
    oidc_issuer_enabled                 = true
    os_disk_size_gb                     = 100
    private_cluster_enabled             = true # 仅允许私有网络访问Kube API
    private_cluster_public_fqdn_enabled = true
    private_dns_zone_id                 = null
    public_network_access_enabled       = false
    role_based_access_control_enabled   = true
    rbac_aad                            = true
    rbac_aad_managed                    = true
    local_account_disabled              = true
    sku_tier                            = "Free" # 生产环境节点超过10个后改为Standard
    temporary_name_for_rotation         = "tmp"  # 长度需小于12字符，仅允许小写字母
    workload_identity_enabled           = true # 使用Azure工作负载身份将AAD身份映射到Pod
    azure_policy_enabled                = false
    network_contributor_role_assigned_subnet_ids = {
      cluster_subnet = local.cluster_subnet_id
    }
    node_pools = {
      spots = {
        name                   = "spots"
        vm_size                = "Standard_D2s_v4"
        enable_auto_scaling    = true
        min_count              = 3
        max_count              = 4
        eviction_policy        = "Delete"
        enable_host_encryption = true
        priority               = "Spot"
        spot_max_price         = "0.50"
        zones                  = [1,2,3]
        node_taints            = ["kubernetes.azure.com/scalesetpriority=spot:NoSchedule"]
        vnet_subnet_id         = local.cluster_subnet_id
      }
    }
  }
}

module "aks" {
  source                                       = "Azure/aks/azurerm"
  version                                      = "7.5.0"
  resource_group_name                          = local.rg_name
  location                                     = local.location
  cluster_name                                 = "aks-test-cluster"
  prefix                                       = "aks-test"
  vnet_subnet_id                               = local.cluster_subnet_id
  identity_type                                = local.aks.identity_type
  agents_availability_zones                    = local.aks.agents_availability_zones
  agents_count                                 = local.aks.agents_count
  agents_size                                  = "Standard_D2s_v4"
  agents_min_count                             = 3
  agents_max_count                             = 3
  enable_auto_scaling                          = local.aks.enable_auto_scaling
  enable_host_encryption                       = local.aks.enable_host_encryption
  key_vault_secrets_provider_enabled           = local.aks.key_vault_secrets_provider_enabled
  kms_enabled                                  = local.aks.kms_enabled
  kms_key_vault_network_access                 = local.aks.kms_key_vault_network_access
  kubernetes_version                           = "1.27.9"
  log_analytics_workspace_enabled              = local.aks.log_analytics_workspace_enabled
  network_plugin                               = local.aks.network_plugin
  net_profile_pod_cidr                         = local.aks.net_profile_pod_cidr
  net_profile_service_cidr                     = local.aks.net_profile_service_cidr
  net_profile_dns_service_ip                   = local.aks.net_profile_dns_service_ip
  net_profile_outbound_type                    = local.aks.net_profile_outbound_type
  oidc_issuer_enabled                          = local.aks.oidc_issuer_enabled
  orchestrator_version                         = local.aks_kubernetes_version
  os_disk_size_gb                              = local.aks.os_disk_size_gb
  private_cluster_enabled                      = local.aks.private_cluster_enabled
  private_cluster_public_fqdn_enabled          = local.aks.private_cluster_public_fqdn_enabled
  private_dns_zone_id                          = local.aks.private_dns_zone_id
  # public_network_access_enabled                = local.aks.public_network_access_enabled
  role_based_access_control_enabled            = local.aks.role_based_access_control_enabled
  rbac_aad                                     = local.aks.rbac_aad
  rbac_aad_managed                             = local.aks.rbac_aad_managed
  rbac_aad_admin_group_object_ids              = local.rbac_aad_admin_group_object_ids
  local_account_disabled                       = local.aks.local_account_disabled
  sku_tier                                     = local.aks.sku_tier
  node_pools                                   = local.aks.node_pools
  azure_policy_enabled                         = local.aks.azure_policy_enabled
  workload_identity_enabled                    = local.aks.workload_identity_enabled
  temporary_name_for_rotation                  = local.aks.temporary_name_for_rotation
  attached_acr_id_map = {

  }
}

内容的提问来源于stack exchange，提问作者Mike Elmsly