Cloud Workflows调用GCE API启动容器时GCR认证失败排查

问题背景 #

• 参考官方指南部署Google Workflow，实现Compute Engine容器优化型VM自动启动、长时任务运行、VM自动清理全流程，配套通过Terraform完成工作流配置。
• 容器镜像由Cloud Build CI/CD流水线构建后上传至Google Container Registry（GCR），本地运行、GCP控制台手动启动容器优化型VM时镜像均可正常加载运行。
• 通过Cloud Workflow控制台触发VM启动时，出现GCR认证相关错误，错误堆栈截图如下：
  
• 已核对配置：控制台手动启动的VM与工作流自动启动的VM使用完全相同的服务账号与权限范围，且已按照指南要求将正常运行VM的等价REST请求转换为YAML配置写入Terraform，控制台对比两类VM配置参数完全一致。

现有配置代码 #

Workflow对应的Terraform配置 #

resource "google_workflows_workflow" "workflows_example" {
  name            = "scraper-workflow"
  region          = "us-central1"
  description     = "Scraper workflow"
  service_account = "scraper-workflow-executor@rdmops-219503.iam.gserviceaccount.com"
  source_contents = <<-EOF
  # FYI, In terraform you need to escape the $$ or it will cause errors.

  - init:
      assign:
      - commitSHA: ${var.SCRAPER_IMAGE_COMMIT_SHA}
      - projectId: $${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
      - projectNumber: $${sys.get_env("GOOGLE_CLOUD_PROJECT_NUMBER")}
      - zone: "us-central1-a"
      - machineType: "e2-medium"
      - ticker: $${args.ticker}
      - instanceName: $${ticker+"-scraper"}
  - create_and_start_vm:
      call: googleapis.compute.v1.instances.insert
      args:
        project: $${projectId}
        zone: $${zone}
        body:
          canIpForward: false
          confidentialInstanceConfig:
            enableConfidentialCompute: false
          deletionProtection: false
          shieldedInstanceConfig:
            enableIntegrityMonitoring: true
            enableSecureBoot: false
            enableVtpm: true
          tags:
            items:
            - http-server
            - https-server
          name: $${instanceName}
          labels:
            - "container-vm": "cos-stable-97-16919-29-40"
          machineType: $${"zones/" + zone + "/machineTypes/" + machineType}
          disks:
          - initializeParams:
              diskSizeGb: "10"
              diskType: "projects/rdmops-219503/zones/us-central1-a/diskTypes/pd-balanced"
              sourceImage: "projects/cos-cloud/global/images/cos-stable-97-16919-29-40"
            boot: true
            autoDelete: true
            deviceName: $${instanceName}
          # 配置VM外部IP
          networkInterfaces:
          - accessConfigs:
            - name: "External NAT"
              networkTier: "PREMIUM"
            stackType: "IPV4_ONLY"
            subnetwork: "projects/rdmops-219503/regions/us-central1/subnetworks/default"
          # 容器运行配置
          metadata:
              items:
              - key: "google-logging-enabled"
                value: "true"
              - key: "gce-container-declaration"
                value: '$${"spec:\n  containers:\n  - name: scraper-workflow\n    image: gcr.io/" + projectId + "/scraper-workflow:" + commitSHA + "\n    stdin: false\n    tty: false\n  restartPolicy: Never\n"}'

          # 容器拉取与运行所需权限配置
          serviceAccounts:
          - email: 937088654099-compute@developer.gserviceaccount.com
            scopes:
            - https://www.googleapis.com/auth/devstorage.read_only
            - https://www.googleapis.com/auth/logging.write
            - https://www.googleapis.com/auth/monitoring.write
            - https://www.googleapis.com/auth/servicecontrol
            - https://www.googleapis.com/auth/service.management.readonly
            - https://www.googleapis.com/auth/trace.append
  - log_wait_for_vm_network:
      call: sys.log
      args:
        data: $${"Waiting for VM network to initialize"}
  - wait_for_vm_network:
      call: sys.sleep
      args:
          seconds: 10
  - get_instance:
      call: googleapis.compute.v1.instances.get
      args:
        instance: $${instanceName}
        project: $${projectId}
        zone: $${zone}
      result: instance
  - extract_external_ip_and_construct_urls:
      assign:
          - external_ip: $${instance.networkInterfaces[0].accessConfigs[0].natIP}
          - base_url: $${"http://" + external_ip + "/"}
          - start_url: $${base_url + "start"}
          - poll_url: $${base_url + "poll"}
  # 其余工作流逻辑已省略
EOF
}

控制台手动创建正常运行实例的等价REST请求 #

POST https://www.googleapis.com/compute/v1/projects/rdmops-219503/zones/us-central1-a/instances
{
  "canIpForward": false,
  "confidentialInstanceConfig": {
    "enableConfidentialCompute": false
  },
  "deletionProtection": false,
  "description": "",
  "disks": [
    {
      "autoDelete": true,
      "boot": true,
      "deviceName": "instance-1",
      "initializeParams": {
        "diskSizeGb": "10",
        "diskType": "projects/rdmops-219503/zones/us-central1-a/diskTypes/pd-balanced",
        "labels": {},
        "sourceImage": "projects/cos-cloud/global/images/cos-stable-97-16919-29-40"
      },
      "mode": "READ_WRITE",
      "type": "PERSISTENT"
    }
  ],
  "displayDevice": {
    "enableDisplay": false
  },
  "guestAccelerators": [],
  "keyRevocationActionType": "NONE",
  "labels": {
    "container-vm": "cos-stable-97-16919-29-40"
  },
  "machineType": "projects/rdmops-219503/zones/us-central1-a/machineTypes/e2-medium-medium",
  "metadata": {
    "items": [
      {
        "key": "ssh-keys",
        "value": "XXXXX"
      },
      {
        "key": "gce-container-declaration",
        "value": "spec:\n  containers:\n  - name: instance-9\n    image: gcr.io/rdmops-219503/workflow-scraper:test3\n    args:\n    - ''\n    stdin: false\n    tty: false\n  restartPolicy: Never\n# This container declaration format is not public API and may change without notice. Please\n# use gcloud command-line tool or Google Cloud Console to run Containers on Google Compute Engine."
      }
    ]
  },
  "name": "instance-9",
  "networkInterfaces": [
    {
      "accessConfigs": [
        {
          "name": "External NAT",
          "networkTier": "PREMIUM"
        }
      ],
      "stackType": "IPV4_ONLY",
      "subnetwork": "projects/rdmops-219503/regions/us-central1/subnetworks/default"
    }
  ],
  "reservationAffinity": {
    "consumeReservationType": "ANY_RESERVATION"
  },
  "scheduling": {
    "automaticRestart": true,
    "onHostMaintenance": "MIGRATE",
    "preemptible": false,
    "provisioningModel": "STANDARD"
  },
  "serviceAccounts": [
    {
      "email": "937088654099-compute@developer.gserviceaccount.com",
      "scopes": [
        "https://www.googleapis.com/auth/devstorage.read_only",
        "https://www.googleapis.com/auth/logging.write",
        "https://www.googleapis.com/auth/monitoring.write",
        "https://www.googleapis.com/auth/servicecontrol",
        "https://www.googleapis.com/auth/service.management.readonly",
        "https://www.googleapis.com/auth/trace.append"
      ]
    }
  ],
  "shieldedInstanceConfig": {
    "enableIntegrityMonitoring": true,
    "enableSecureBoot": false,
    "enableVtpm": true
  },
  "tags": {
    "items": [
      "http-server",
      "https-server"
    ]
  },
  "zone": "projects/rdmops-219503/zones/us-central1-a"
}

根因定位 #

对比两份配置可直接定位三个核心问题：

• labels字段格式错误：Workflow配置中labels错误写成了列表格式，而GCE API要求labels为键值对对象格式，错误结构会导致元数据解析异常，干扰容器运行时的凭证读取逻辑。
• 容器镜像名称不匹配：手动创建的正常实例拉取的镜像是gcr.io/rdmops-219503/workflow-scraper:test3，但Workflow中拼接的镜像路径为gcr.io/${projectId}/scraper-workflow:${commitSHA}，镜像名称顺序写反，直接触发拉取失败，报错表现和GCR认证错误一致。
• gce-container-declaration元数据缺失固定尾部声明：容器优化型OS的容器代理解析元数据时，会校验该字段尾部的固定提示注释，缺失该段内容会导致容器运行时初始化异常，无法正确读取VM绑定的服务账号凭证拉取镜像。

修复方案 #

按以下步骤修改Terraform配置即可解决问题：

• 修正labels字段为对象格式，移除键值对前的列表前缀-
• 核对镜像路径，将镜像名称改为和Cloud Build推送一致的workflow-scraper，确认变量拼接后的完整镜像路径和手动测试时的路径规则一致
• 补全gce-container-declaration字段尾部的固定注释内容，和控制台生成的元数据结构保持完全一致
• 提前确认VM绑定的计算引擎默认服务账号已授予GCR镜像拉取权限，避免元数据注入阶段出现权限校验失败。

内容的提问来源于stack exchange，提问作者Zach