在Databricks中用Python创建Azure Data Lake Gen2容器报错求助

报错信息 #

两段代码均触发相同错误：

AzureSigningError: Invalid base64-encoded string: number of data characters (37) cannot be 1 more than a multiple of 4

尝试的代码 #

第一段代码（DataLakeServiceClient方式） #

from azure.storage.filedatalake import DataLakeServiceClient

service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format(
           "https", storage_account_name), credential=my_secret_key)

#the get_file_system_client method will not throw error if the file system does not exist, if you're using the latest library 12.3.0
file_system_client = service_client.get_file_system_client("mytestfolder")

print("the file system exists: " + str(file_system_client.exists()))

#create the file system if it does not exist
if not file_system_client.exists():
    file_system_client.create_file_system()
    print("the file system is created.")

第二段代码（BlobServiceClient方式） #

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
container_name = 'test'
storage_connection_string = f'DefaultEndpointsProtocol=https;AccountName={acname};AccountKey={ackey};EndpointSuffix= core.windows.net;'
blob_service_client = BlobServiceClient.from_connection_string(storage_connection_string)
blob_service_client.create_container(container_name,public_access='CONTAINER',timeout = 10)

问题原因 #

1. 存储账户密钥格式错误：存储账户密钥是标准Base64编码字符串，长度必须为4的倍数。报错提示字符数37，说明密钥存在拼写错误、多余空格或字符遗漏。
2. 连接字符串格式错误：第二段代码中EndpointSuffix= core.windows.net;的core.windows.net前存在多余空格，会导致连接字符串解析失败。

修复方案 #

方案1：修正连接字符串与密钥（BlobServiceClient方式） #

from azure.storage.blob import BlobServiceClient

container_name = 'test'
# 去掉EndpointSuffix后的空格，确保ackey是完整正确的Base64格式密钥
storage_connection_string = f'DefaultEndpointsProtocol=https;AccountName={acname};AccountKey={ackey};EndpointSuffix=core.windows.net;'
blob_service_client = BlobServiceClient.from_connection_string(storage_connection_string)

# 先检查容器是否存在，避免重复创建报错
container_client = blob_service_client.get_container_client(container_name)
if not container_client.exists():
    blob_service_client.create_container(container_name, public_access='CONTAINER', timeout=10)
    print(f"容器 {container_name} 创建成功")
else:
    print(f"容器 {container_name} 已存在")

方案2：修正密钥后的DataLakeServiceClient写法 #

from azure.storage.filedatalake import DataLakeServiceClient

# 确保storage_account_name和my_secret_key为正确的账户名与Base64格式密钥
service_client = DataLakeServiceClient(
    account_url=f"https://{storage_account_name}.dfs.core.windows.net",
    credential=my_secret_key
)

file_system_name = "mytestfolder"
file_system_client = service_client.get_file_system_client(file_system_name)

if not file_system_client.exists():
    file_system_client.create_file_system()
    print(f"文件系统 {file_system_name} 创建成功")
else:
    print(f"文件系统 {file_system_name} 已存在")

方案3：Databricks PySpark直接创建（服务主体认证） #

# 替换<>中的参数为实际值
storage_account = "<storage-account-name>"
client_id = "<service-principal-client-id>"
client_secret = "<service-principal-secret>"
tenant_id = "<azure-tenant-id>"
container_name = "test-container"

# 配置Spark服务主体认证
spark.conf.set(f"fs.azure.account.auth.type.{storage_account}.dfs.core.windows.net", "OAuth")
spark.conf.set(f"fs.azure.account.oauth.provider.type.{storage_account}.dfs.core.windows.net", "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider")
spark.conf.set(f"fs.azure.account.oauth2.client.id.{storage_account}.dfs.core.windows.net", client_id)
spark.conf.set(f"fs.azure.account.oauth2.client.secret.{storage_account}.dfs.core.windows.net", client_secret)
spark.conf.set(f"fs.azure.account.oauth2.client.endpoint.{storage_account}.dfs.core.windows.net", f"https://login.microsoftonline.com/{tenant_id}/oauth2/token")

# 创建容器（不存在则自动创建）
adls_path = f"abfss://{container_name}@{storage_account}.dfs.core.windows.net/"
dbutils.fs.mkdirs(adls_path)
print(f"容器 {container_name} 创建成功（或已存在）")

内容的提问来源于stack exchange，提问作者Pysparker