Terraform部署Azure Data Collection Rule时触发400无效请求错误

问题概述 #

使用Terraform批量创建Azure Data Collection Rule（DCR）时，执行terraform apply触发400 Bad Request错误，提示InvalidPayload: Data collection rule is invalid，两个规则（rule1、rule2）均创建失败。

错误信息 #

Error: creating Data Collection Rule (Subscription: "***"
│ Resource Group Name: "rg-management-prd-cus-001"
│ Data Collection Rule Name: "dcr-monitoring-prd-cus-001"): unexpected status 400 (400 Bad Request) with error: InvalidPayload: Data collection rule is invalid
│ 
│   with module.management_subscription[0].azurerm_monitor_data_collection_rule.amdcr["rule1"],
│   on subscriptions/management/management.tf line 751, in resource "azurerm_monitor_data_collection_rule" "amdcr":
│  751: resource "azurerm_monitor_data_collection_rule" "amdcr" {
│ 
│ creating Data Collection Rule (Subscription:
│ "***"
│ Resource Group Name: "rg-management-prd-cus-001"
│ Data Collection Rule Name: "dcr-monitoring-prd-cus-001"): unexpected status
│ 400 (400 Bad Request) with error: InvalidPayload: Data collection rule is
│ invalid
╵
╷
│ Error: creating Data Collection Rule (Subscription: "***"
│ Resource Group Name: "rg-management-prd-cus-001"
│ Data Collection Rule Name: "dcr-monitoring-prd-cus-002"): unexpected status 400 (400 Bad Request) with error: InvalidPayload: Data collection rule is invalid
│ 
│   with module.management_subscription[0].azurerm_monitor_data_collection_rule.amdcr["rule2"],
│   on subscriptions/management/management.tf line 751, in resource "azurerm_monitor_data_collection_rule" "amdcr":
│  751: resource "azurerm_monitor_data_collection_rule" "amdcr" {
│ 
│ creating Data Collection Rule (Subscription:
│ "***"
│ Resource Group Name: "rg-management-prd-cus-001"
│ Data Collection Rule Name: "dcr-monitoring-prd-cus-002"): unexpected status
│ 400 (400 Bad Request) with error: InvalidPayload: Data collection rule is
│ invalid

相关代码 #

Local块 #

locals {
  data_collection_rule = {
   rule1 = {
     name = "001"
     data_flow = {
        streams = ["Microsoft-Event"]
        destinations= ["la--210586112"]
        transform_kql = "source"
        output_stream = "Microsoft-Event"
     }
      data_sources = {
        Windows_Event_Log = {
          streams        = ["Microsoft-WindowsEvent"]
          x_path_queries = ["Application!*[System[(Level=1 or Level=2 or Level=3 or Level=4 or Level=0)]]",
                            "Security!*[System[(band(Keywords,13510798882111488))]]",
                            "System!*[System[(Level=1 or Level=2 or Level=3 or Level=4 or Level=0)]]"]
          name           =  "eventLogsDataSource"
        }

      }
   }

   rule2 = {
    name = "002"
    data_flow = {
      streams = ["Microsoft-Syslog"]
      destinations= ["la--210586112"]
      transform_kql = "source"
      output_stream = "Microsoft-Syslog"

    }
    data_sources = {
      SysLog = {
        streams        = ["Microsoft-Syslog"]
        facility_names = [    
                                "alert",
                                "audit",
                                "auth",
                                "authpriv",
                                "clock",
                                "cron",
                                "daemon",
                                "ftp",
                                "kern",
                                "local0",
                                "local1",
                                "local2",
                                "local3",
                                "local4",
                                "local5",
                                "local6",
                                "local7",
                                "lpr",
                                "mail",
                                "news",
                                "nopri",
                                "ntp",
                                "syslog",
                                "user",
                                "uucp"
                              ]
        name           = "sysLogsDataSource-1688419672"
        log_levels     = "*"
      }
      }
    }

  }
}

Resource块 #

resource "azurerm_monitor_data_collection_rule" "amdcr" {
  for_each                    = local.data_collection_rule
  name                        = format("dcr-monitoring-%s-%s-%s",var.environment,var.location_short_name,each.value.name)
  resource_group_name         = module.resource_group.rg_name_subs
  location                    = var.location
  data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.monitor_dce.id

  destinations {
    log_analytics {
      workspace_resource_id = module.log_analytics_workspace.id
      name                  = "dataCollectionRules_${each.value.name}_law"
    }
  }

  dynamic "data_flow" {
    for_each = each.value.data_flow != null ? [each.value.data_flow] : []
    content {
      streams        = data_flow.value.streams
      destinations   = data_flow.value.destinations
      transform_kql  = data_flow.value.transform_kql
      output_stream  = data_flow.value.output_stream
    }
  }

  dynamic "data_sources" {
    for_each = each.value.data_sources != null ? each.value.data_sources : {}
    content {
      dynamic "windows_event_log" {
        for_each = contains(keys(data_sources.value), "Windows_Event_Log") ? [1] : []
        content {
          streams        = lookup(data_sources.value.Windows_Event_Log, "streams", [])
          x_path_queries = lookup(data_sources.value.Windows_Event_Log, "x_path_queries", [])
          name           = lookup(data_sources.value.Windows_Event_Log, "name", "")
        }
      }

      dynamic "syslog" {
        for_each = contains(keys(data_sources.value), "SysLog") ? [1] : []
        content {
          streams        = lookup(data_sources.value.SysLog, "streams", [])
          facility_names = lookup(data_sources.value.SysLog, "facility_names", [])
          log_levels     = lookup(data_sources.value.SysLog, "log_levels", "*")
          name           = lookup(data_sources.value.SysLog, "name", "")
        }
      }
    }
  }
}

错误原因分析 #

1. 数据流目标名称不匹配：Local块中data_flow.destinations指定的"la--210586112"，和Resource块中destinations.log_analytics.name生成的"dataCollectionRules_${each.value.name}_law"完全不一致。Azure要求数据流中的目标名称必须和规则定义的目标名称严格对应。

2. 数据流输入流与数据源输出流不匹配：rule1的data_flow.streams是["Microsoft-Event"]，但对应的Windows Event Log数据源输出流是["Microsoft-WindowsEvent"]，两者不匹配导致数据流无法正确关联数据源。

3. Data Sources块结构错误：Resource块中使用dynamic "data_sources"循环遍历each.value.data_sources，会为每个数据源类型生成一个独立的data_sources块，但Azure DCR要求所有数据源必须放在同一个data_sources块内。

4. Syslog日志级别类型错误：Local块中SysLog.log_levels是字符串"*"，而Terraform Azure Provider要求该字段为列表类型（如["*"]）。

解决方法 #

1. 修正数据流目标名称与输入流 #

更新Local块中data_flow的配置，使其与数据源输出流、目标名称匹配：

locals {
  data_collection_rule = {
   rule1 = {
     name = "001"
     data_flow = {
        streams = ["Microsoft-WindowsEvent"] # 同步数据源的输出流
        destinations= ["dataCollectionRules_001_law"] # 匹配目标名称
        transform_kql = "source"
        output_stream = "Microsoft-WindowsEvent"
     }
     # ... 其余代码不变
   }

   rule2 = {
    name = "002"
    data_flow = {
      streams = ["Microsoft-Syslog"]
      destinations= ["dataCollectionRules_002_law"] # 匹配目标名称
      transform_kql = "source"
      output_stream = "Microsoft-Syslog"
    }
    # ... 其余代码不变
   }
  }
}

2. 修正Data Sources块结构 #

删除Resource块中的dynamic "data_sources"，替换为单个data_sources块，内部动态生成不同类型的数据源：

resource "azurerm_monitor_data_collection_rule" "amdcr" {
  # ... 其余代码不变

  data_sources {
    dynamic "windows_event_log" {
      for_each = lookup(each.value.data_sources, "Windows_Event_Log", {}) != {} ? [each.value.data_sources.Windows_Event_Log] : []
      content {
        streams        = windows_event_log.value.streams
        x_path_queries = windows_event_log.value.x_path_queries
        name           = windows_event_log.value.name
      }
    }

    dynamic "syslog" {
      for_each = lookup(each.value.data_sources, "SysLog", {}) != {} ? [each.value.data_sources.SysLog] : []
      content {
        streams        = syslog.value.streams
        facility_names = syslog.value.facility_names
        log_levels     = syslog.value.log_levels # 后续需修正为列表
        name           = syslog.value.name
      }
    }
  }
}

3. 修正Syslog日志级别类型 #

更新Local块中SysLog.log_levels为列表类型：

locals {
  data_collection_rule = {
   rule2 = {
    # ... 其余代码不变
    data_sources = {
      SysLog = {
        # ... 其余代码不变
        log_levels     = ["*"] # 从字符串改为列表
      }
    }
   }
  }
}

4. 验证配置 #

执行terraform plan检查配置是否合法，确认无语法错误后再执行terraform apply。

内容的提问来源于stack exchange，提问作者Pallab