Azure DevOps流水线生成Databricks令牌失败，请求代码排查

初始代码及问题 #

代码 #

variables:
  azureSubscription: 
  AZURE_DATABRICKS_APP_ID: 2f0e6f879c1d
  rg: 
  databricksName: 

stages:
- stage: databricksDeployment
  jobs:
  - job: DBW
    pool:
      vmImage: 'ubuntu-latest'
    steps:
    - task: AzureCLI@2
      inputs:
        azureSubscription: $(azureSubscription)
        addSpnToEnvironment: true
        scriptType: 'pscore'
        scriptLocation: 'inlineScript'
        inlineScript: |
          #databrick cli
          # install databrick-cli
          python -m pip install --upgrade pip setuptools wheel databricks-cli
          
          $wsId=(az resource show --resource-type Microsoft.Databricks/workspaces -g $(rg) -n $(databricksName) --query id -o tsv)
          $workspaceUrl=(az resource show --resource-type Microsoft.Databricks/workspaces -g $(rg) -n $(databricksName) --query properties.workspaceUrl --output tsv)

          $workspaceUrlPost='https://'
          $workspaceUrlPost+=$workspaceUrl
          $workspaceUrlPost+='/api/2.0/token/create'
          echo "Https Url with Post: $workspaceUrlPost"

          $workspaceUrlHttps='https://'
          $workspaceUrlHttps+=$workspaceUrl
          $workspaceUrlHttps+='/'
          echo "Https Url : $workspaceUrlHttps"

          # token response for the Azure Databricks app
          $token=(az account get-access-token --resource $(AZURE_DATABRICKS_APP_ID) --query "accessToken" --output tsv)
          echo "Token retrieved: $token"

          # Get a token for the Azure management API
          $azToken=(az account get-access-token --resource https://management.core.windows.net/ --query "accessToken" --output tsv)

          # Create PAT token valid for approximately 10 minutes (600 seconds). Note the quota limit of 600 tokens.
          $pat_token_response=(curl --insecure -X POST ${workspaceUrlPost} `
            -H "Authorization: Bearer $token" `
            -H "X-Databricks-Azure-SP-Management-Token:$azToken" `
            -H "X-Databricks-Azure-Workspace-Resource-Id:$wsId" `
            -d '{"lifetime_seconds": 600,"comment": "this is an example token"}')
          
          # Print PAT token
          $dapiToken=($pat_token_response | ConvertFrom-Json).token_value
          echo "DATABRICKS_TOKEN: $dapiToken"

          $env:DATABRICKS_HOST=$workspaceUrlHttps
          $env:DATABRICKS_TOKEN=$dapiToken
          echo "DATABRICKS_HOST: $workspaceUrlHttps"
          echo "DATABRICKS_TOKEN: $dapiToken"
      displayName: 'Run after deployment script'

    - script: |
        echo "Starting Databricks notebook upload..."
        # Install Databricks CLI
        pip install databricks-cli
        
        # Set Databricks Host and Token
        export DATABRICKS_HOST="$DATABRICKS_HOST"
        export DATABRICKS_TOKEN="$DATABRICKS_TOKEN"

        # Authenticate with Databricks using the PAT
        echo "Authenticating with Databricks..."
        databricks configure --token <<EOF
        $DATABRICKS_HOST
        $DATABRICKS_TOKEN
        EOF

错误信息 #

Databricks Host (should begin with https://): Databricks Host (should begin with https://): Databricks Host (should begin with https://): Aborted!

更新后代码及问题 #

代码 #

variables:
  armConnection: 
  resourceID: 
  resourceGroup: 
  databricksName: 

trigger:
- development

pool:
  vmImage: 'ubuntu-latest'
  
steps:
- task: UsePythonVersion@0
  inputs:
    versionSpec: '3.x'
    addToPath: true

- checkout: self

- task: AzureCLI@2
  displayName: 'Azure CLI '
  inputs:
    azureSubscription: $(armConnection)
    scriptType: pscore
    scriptLocation: inlineScript
    inlineScript: |
     # Get Azure Databricks workspace information
     $workspaceUrl = 'https://'
     $workspaceUrl += (az resource show --resource-type Microsoft.Databricks/workspaces -g $(resourceGroup) -n $(databricksName) --query properties.workspaceUrl --output tsv)
     Write-Host "Workspace Url: $workspaceUrl"
     
     # Generate AAD Access Token for Azure Databricks service.
     $token = (az account get-access-token --resource $(resourceID) --query "accessToken" --output tsv)
     Write-Host "AAD Access Token: $token"
     
     # Generate Azure Databricks PAT.
     $adbPAT_response = (curl --request POST "$workspaceUrl/api/2.0/token/create" --header "Authorization: Bearer $token" --data '{"lifetime_seconds": 600, "comment": "This is an example token."}')
     Write-Host $adbPAT_response
     $env:DATABRICKS_HOST=$workspaceUrl
     $env:DATABRICKS_TOKEN=$adbPAT_response
     echo "DATABRICKS_HOST: $workspaceUrl"
     echo "DATABRICKS_TOKEN: $adbPAT_response"

- script: |
        echo "Starting Databricks notebook upload..."
        # Install Databricks CLI
        pip install databricks-cli
        
        # Set Databricks Host and Token
        export DATABRICKS_HOST="$DATABRICKS_HOST"
        export DATABRICKS_TOKEN="$DATABRICKS_TOKEN"

        # Authenticate with Databricks using the PAT
        echo "Authenticating with Databricks..."
        databricks configure --token <<EOF
        workspaceUrl
        $DATABRICKS_TOKEN
        
        EOF
        # Specify the full paths to the source files
        common_function_path=$(Build.SourcesDirectory)/notebook.sql
        
        # Specify the full target paths in Databricks
        common_function_target_path="/Users/user/notebook.sql"
        
        # Upload notebooks to Databricks workspace
        echo "Uploading notebooks to Databricks..."
        databricks workspace import --language sql --overwrite "$common_function_path" "$common_function_target_path"
        
        echo "Notebooks uploaded successfully."
  displayName: 'Upload Databricks Notebooks'   

错误信息 #

Authenticating with Databricks...
Databricks Host (should begin with https://): Token:/opt/hostedtoolcache/Python/3.11.6/x64/lib/python3.11/getpass.py:91: GetPassWarning: Can not control echo on the terminal.
  passwd = fallback_getpass(prompt, stream)
Warning: Password input may be echoed.
 
Warning: Password input may be echoed.
 
Token:Token:
Warning: Password input may be echoed.
 Aborted!
Uploading notebooks to Databricks...
Error: IndexError: string index out of range

问题排查及修复方案 #

核心问题点 #

1. 环境变量跨任务传递失败：AzureCLI@2任务中用$env:设置的是PowerShell会话内的环境变量，无法传递到后续的bash脚本任务，必须使用Azure DevOps流水线级变量传递。
2. PowerShell中curl命令误用：PowerShell里的curl是Invoke-WebRequest的别名，参数格式与bash的curl不同，导致PAT生成请求失败，返回无效响应。
3. PAT响应未正确解析：两次代码都未正确提取响应中的token_value字段，第二次直接将整个JSON响应作为令牌，导致认证失败。
4. bash脚本变量引用错误：更新后的代码中databricks configure部分写死了workspaceUrl字符串，未引用变量$DATABRICKS_HOST，导致主机地址为空。

修复后的代码示例 #

variables:
  armConnection: 
  resourceGroup: 
  databricksName: 

trigger:
- development

pool:
  vmImage: 'ubuntu-latest'
  
steps:
- task: UsePythonVersion@0
  inputs:
    versionSpec: '3.x'
    addToPath: true

- checkout: self

- task: AzureCLI@2
  displayName: 'Generate Databricks PAT'
  inputs:
    azureSubscription: $(armConnection)
    scriptType: pscore
    scriptLocation: inlineScript
    inlineScript: |
     # 获取Databricks工作区URL
     $workspaceUrl = "https://$(az resource show --resource-type Microsoft.Databricks/workspaces -g $(resourceGroup) -n $(databricksName) --query properties.workspaceUrl --output tsv)"
     Write-Host "Workspace Url: $workspaceUrl"
     
     # 获取AAD访问令牌（Databricks资源ID固定为2ff814a6-3304-4ab8-85cb-cd0e6f879c1d）
     $aadToken = (az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --query "accessToken" --output tsv)
     Write-Host "AAD Access Token retrieved"
     
     # 获取Azure管理API令牌
     $managementToken = (az account get-access-token --resource https://management.core.windows.net/ --query "accessToken" --output tsv)
     
     # 获取工作区资源ID
     $workspaceResourceId = (az resource show --resource-type Microsoft.Databricks/workspaces -g $(resourceGroup) -n $(databricksName) --query id --output tsv)
     
     # 使用Invoke-RestMethod发送PAT生成请求
     $patRequestBody = @{
       lifetime_seconds = 600
       comment = "Pipeline generated token for notebook upload"
     } | ConvertTo-Json
     
     try {
       $patResponse = Invoke-RestMethod -Uri "$workspaceUrl/api/2.0/token/create" `
         -Method Post `
         -Headers @{
           "Authorization" = "Bearer $aadToken"
           "X-Databricks-Azure-SP-Management-Token" = $managementToken
           "X-Databricks-Azure-Workspace-Resource-Id" = $workspaceResourceId
         } `
         -Body $patRequestBody `
         -ContentType "application/json"
       
       $databricksToken = $patResponse.token_value
       Write-Host "Databricks PAT generated successfully"
       
       # 设置流水线任务变量，供后续任务使用
       Write-Host "##vso[task.setvariable variable=DATABRICKS_HOST;]$workspaceUrl"
       Write-Host "##vso[task.setvariable variable=DATABRICKS_TOKEN;]$databricksToken"
     } catch {
       Write-Host "Error generating PAT: $_"
       exit 1
     }

- script: |
    echo "Starting Databricks notebook upload..."
    # 安装Databricks CLI
    pip install databricks-cli
    
    # 直接使用流水线变量，无需额外export
    echo "Authenticating with Databricks..."
    # 使用非交互式配置，避免输入提示
    databricks configure --token --host "$DATABRICKS_HOST" <<EOF
    $DATABRICKS_TOKEN
    EOF
    
    # 上传Notebook
    common_function_path=$(Build.SourcesDirectory)/notebook.sql
    common_function_target_path="/Users/user/notebook.sql"
    
    echo "Uploading notebooks to Databricks..."
    databricks workspace import --language sql --overwrite "$common_function_path" "$common_function_target_path"
    
    echo "Notebooks uploaded successfully."
  displayName: 'Upload Databricks Notebooks'   

关键修复说明 #

• 变量传递：用Write-Host "##vso[task.setvariable variable=XXX;]$value"设置流水线级变量，确保后续bash任务能读取到。
• PAT生成：改用PowerShell的Invoke-RestMethod替代curl，参数格式符合PowerShell语法，同时添加错误捕获便于排查问题。
• 非交互式配置：databricks configure使用--host参数指定主机地址，减少输入步骤，避免变量引用错误。
• 固定资源ID：Databricks的AAD资源ID为固定值2ff814a6-3304-4ab8-85cb-cd0e6f879c1d，无需自定义变量。

内容的提问来源于stack exchange，提问作者Vin