Terraform配置Azure Databricks Provider创建Metastore报错

问题背景 #

在Azure DevOps中有两条Terraform v1.5.3流水线：

• 第一条流水线：创建Azure Databricks工作区、DataLake存储、Databricks访问连接器，并为连接器分配DataLake的Storage Blob Data Contributor角色，状态文件存储在指定Azure存储容器。
• 第二条流水线：通过remote_state.tf读取第一条流水线的状态，登录Databricks并创建Metastore，但执行时触发权限报错。

报错信息 #

│ Error: cannot create metastore: Only account admin can create metastores.
│
│   with databricks_metastore.metastore,
│   on resources.tf line 1, in resource "databricks_metastore" "metastore":
│    1: resource "databricks_metastore" "metastore" {
│╵

已验证权限 #

使用的服务主体已具备：

• Active Directory全局管理员身份
• Azure订阅级别Owner、Storage Blob Data Owner、User Access Administrator角色

流水线代码 #

第一条流水线Terraform代码 #

### create Resource Group
module "resource_group" {
  source = "./modules/Module-ResourceGroup-v1"

  resource_group_name = var.resource_group_name
  location            = var.location
}

# create Azure Data Bricks Service
module "adb" {
  
  source = "./modules/Module-DataBrickWorkSpace-v1"
  
  databrick_workspace_name    = var.databrick_workspace_name
  resource_group_name         = module.resource_group.resource_group_name
  resource_group_location     = module.resource_group.resource_group_location
  managed_resource_group_name = var.managed_resource_group_name
}

# create Azure Data Lake
module "adc-uc-storage" {
  source                      = "./modules/Module-StorageForUnityCatalog-v1"

  uc_storage_account_name     = var.uc_storage_account_name
  resource_group_name         = module.resource_group.resource_group_name
  resource_group_location     = module.resource_group.resource_group_location
  uc_container_name           = var.uc_container_name
}

# creating access connector
module "adb-access-connector" {
  source                      = "./modules/Module-DataBrickAccessConnector-v1"

  access_connector_name       = var.access_connector_name
  location                    = module.resource_group.resource_group_location 
  resource_group_name         = module.resource_group.resource_group_name
  storage_account_id          = module.adc-uc-storage.storage_account_id
}

output "databricks_workspace_resource_id" {
  description = " Resource ID of databrick workspace"
  value = module.adb.databricks_workspace_resource_id
}

output "databrick_workspace_url" {
  description = "URL of databrick workspace"
  value = module.adb.databricks_workspace_url
}

output "databrick_host" {
  description = "The host part of URL of the Databricks workspace"
  value = module.adb.databricks_host
}

output "databricks_workspace_id" {
  description = "The ID of the Databricks workspace"
  value = module.adb.databricks_workspace_id
}
######################################################
output "resource_group_name" {
  description = "name of the adb resource group"
  value = module.resource_group.resource_group_name
}

output "resource_group_id" {
  description = "id of resource group"
  value = module.resource_group.resource_group_id
}

output "resource_group_location" {
 description = "location of resource group"
 value = module.resource_group.resource_group_location
}
###################################################
output "storage_account_id" {
  description = "The ID of the storage account"
  value       = module.adc-uc-storage.storage_account_id
}

output "storage_account_name" {
  description = "The name of the storage account"
  value       = module.adc-uc-storage.storage_account_name
}

output "storage_container_name" {
  description = "The name of the storage container"
  value       = module.adc-uc-storage.storage_container_name
}
#
output "storage_account_principal_id" {
  description = "The principal ID for the storage account"
  value       = module.adc-uc-storage.storage_account_principal_id
}
#
#############################################################
#
output "access_connector_unity_catalog_principal_id" {
  description = "The identity of the Databricks Access Connector"
  value       = module.adb-access-connector.access_connector_unity_catalog_principal_id
}
#
output "access_connector_unity_catalog_resource_id" {
  description = "The ID of the Databricks Access Connector"
  value       = module.adb-access-connector.access_connector_unity_catalog_resource_id
}
#
output "access_connector_name" {
  description = "The name of the Databricks Access Connector"
  value       = module.adb-access-connector.access_connector_name
}
#
output "access_connector_resource_group_name" {
  description = "The name of the resource group where the Databricks Access Connector is created"
  value       = module.adb-access-connector.access_connector_resource_group_name
}
#
output "access_connector_location" {
  description = "The location of the Databricks Access Connector"
  value       = module.adb-access-connector.access_connector_location
}

第二条流水线Terraform代码 #

#remote_state.tf
data "terraform_remote_state" "dev" {
  backend = "azurerm"
  config = {
    resource_group_name  = "pipeline1-storageblob-resourcegroup"
    storage_account_name = "pipeline1storageblob"
    access_key           = "SECRET"
    # this is primary access key of storage account

    container_name = "pipeline1-storageblob-container"
    key            = "pipeline1-state-file"
  }
}

#main.tf 
terraform {
  required_providers {
    azurerm = {
      source = "hashicorp/azurerm"
      version = "~>3.0"
    }
    databricks = {
      source = "databricks/databricks"
       version = "1.21.0"
    }
  }
}

provider "azurerm" {
  features {}
  subscription_id   = "XXXXXXXXXXXXXXXXXXXXXXX"
  tenant_id         = "YYYYYYYYYYYYYYYYYYYYYYY"
  client_id         = "CCCCCCCCCCCCCCCCCCCCCCC"
  client_secret     = "SSSSSSSSSSSSSSSSSSSSSSS"
}

provider "databricks" {
  azure_workspace_resource_id = data.terraform_remote_state.dev.outputs.databricks_workspace_resource_id # this is able to see from output of pipeline 1
  azure_client_id             = "CCCCCCCCCCCCCCCCCCCCCCC"
  azure_client_secret         = "SSSSSSSSSSSSSSSSSSSSSSS"
  azure_tenant_id             = "YYYYYYYYYYYYYYYYYYYYYYY"
}
#resource.tf
resource "databricks_metastore" "metastore" {
  name          = "metastore-allenv"
  storage_root  = format(
                          "abfss://%s@%s.dfs.core.windows.net/",
                          data.terraform_remote_state.dev.outputs.storage_container_name,
                          data.terraform_remote_state.dev.outputs.storage_account_name
                         )
  force_destroy = true
}

解决方案 #

1. 授予服务主体Databricks账户管理员权限 #

Databricks Metastore是账户级资源，仅账户管理员有权创建，需完成以下操作：

• 登录Azure Databricks账户控制台
• 左侧导航栏进入「用户与组」→「服务主体」，将目标服务主体添加到账户中（若未存在）
• 进入「账户管理员」组，将该服务主体加入组内

2. 调整Databricks Provider为账户级认证 #

原Provider配置使用azure_workspace_resource_id，仅能操作工作区级资源，需修改为账户级认证：

provider "databricks" {
  azure_tenant_id       = "YYYYYYYYYYYYYYYYYYYYYYY"
  azure_client_id       = "CCCCCCCCCCCCCCCCCCCCCCC"
  azure_client_secret   = "SSSSSSSSSSSSSSSSSSSSSSS"
  databricks_account_id = "YOUR_DATABRICKS_ACCOUNT_ID" # 可从Databricks账户控制台获取，或在第一条流水线添加该输出
}

3. 补充第一条流水线的Databricks账户ID输出（可选） #

若希望通过remote_state自动读取账户ID，需在第一条流水线的adb模块中添加输出：

output "databricks_account_id" {
  description = "ID of the Databricks account associated with the workspace"
  value = module.adb.databricks_account_id
}

确保Module-DataBrickWorkSpace-v1模块已返回该值，可通过Azure Databricks资源的account_id属性获取。

内容的提问来源于stack exchange，提问作者amit suneja