Google Cloud Scheduler调用Cloud Run容器触发503错误求助

我将容器化的R应用部署到Google Cloud Run，配置Cloud Scheduler通过HTTP请求定期触发代码执行，但收到503错误。本地运行R应用并发送API请求正常，问题出在Cloud Run与外部请求的交互上。

错误日志 #

Cloud Scheduler执行日志 #

{
  httpRequest: {
    status: 503
  }
  insertId: "qx6q58f4iewwp"
  jsonPayload: {
    @type: "type.googleapis.com/google.cloud.scheduler.logging.AttemptFinished"
    jobName: "projects/PROJECT_ID/locations/europe-west1/jobs/scheduled-cloud-run-job"
    status: "UNAVAILABLE"
    targetType: "HTTP"
    url: "CONTAINER_URL"
  }
  logName: "projects/PROJECT_ID/logs/cloudscheduler.googleapis.com%2Fexecutions"
  receiveTimestamp: "2023-06-06T13:20:05.745924313Z"
  resource: {
    labels: {
      job_id: "scheduled-cloud-run-job"
      location: "europe-west1"
      project_id: "PROJECT_ID"
    }
    type: "cloud_scheduler_job"
  }
  severity: "ERROR"
  timestamp: "2023-06-06T13:20:05.745924313Z"
}

Cloud Run请求日志（curl触发） #

{
  httpRequest: {
    latency: "2.733624s"
    protocol: "H2C"
    remoteIp: "X.X.X.X"
    requestMethod: "GET"
    requestSize: "544"
    requestUrl: "<URL>a.run.app"
    responseSize: "1238"
    serverIp: "X.X.X.X"
    status: 503
    userAgent: "curl/7.74.0"
  }
  insertId: "XXXXXXXXXXXXXXXX"
  labels: {
    instanceId: "XXXXXXXXXXXXX"
  }
  logName: "projects/PROJECT_ID/logs/run.googleapis.com%2Frequests"
  receiveTimestamp: "2023-06-07T07:01:14.296557338Z"
  resource: {
    labels: {
      configuration_name: "containerized-pipeline"
      location: "europe-west1"
      project_id: "PROJECT_ID"
      revision_name: "containerized-pipeline-00001-pwn"
      service_name: "containerized-pipeline"
    }
    type: "cloud_run_revision"
  }
  severity: "ERROR"
  spanId: "12660848597156063613"
  textPayload: "The request failed because either the HTTP response was malformed or connection to the instance had an error."
  timestamp: "2023-06-07T07:01:11.486699Z"
  trace: "projects/PROJECT_ID/traces/34366480e3a4143d4008eed0452eacf0"
  traceSampled: true
}

配置文件 #

Cloud Scheduler Terraform配置 #

# -- Create cloud scheduler job -- #
resource "google_cloud_scheduler_job" "default" {
  name             = "scheduled-cloud-run-job"
  description      = "Invokes the Cloud Run container with our pipeline on a recurrent basis."
  schedule         = "*/10 * * * *"
  time_zone        = "Europe/Stockholm"
  retry_config {
    retry_count = 1
  }
  http_target {
    http_method = "GET"
    uri         = "${google_cloud_run_service.default.status[0].url}"
    #body        = base64encode("{\run_container\": \"run\"}")
    #headers     = {"Content-Type" : "application/json", "User-Agent" : "Google-Cloud-Scheduler"}
    oidc_token {
      service_account_email = google_service_account.default.email
    }
  }
}

Cloud Run Terraform配置 #

resource "google_cloud_run_service" "default" {
    name     = "containerized-pipeline"
    location = var.region
    project  = var.project_id
    template {
      spec {
        containers {
          image = "${local.artifact_storage_address}:${local.tag}"
          ports {
            #name           = "h2c"
            container_port = 8080
          }
          resources {
            limits = {
              "cpu"    = "1000m"
              "memory" = "2000Mi"
            }
          }
        }
        container_concurrency = 1
      }
      metadata {
        annotations = {
          "run.googleapis.com/client-name"      = "terraform"
          "autoscaling.knative.dev/minScale"    = 1
          "autoscaling.knative.dev/maxScale"    = 30
          # "run.googleapis.com/cpu-throttling"   = false
        }
      }
    }
    traffic {
        percent         = 100
        latest_revision = true
    }
    depends_on = [
        null_resource.docker_build
    ]
 }

data "google_iam_policy" "noauth" {
   binding {
     role = "roles/run.invoker"
     members = ["allUsers"]
   }
 }

 resource "google_cloud_run_service_iam_policy" "noauth" {
   location    = google_cloud_run_service.default.location
   project     = google_cloud_run_service.default.project
   service     = google_cloud_run_service.default.name
   policy_data = data.google_iam_policy.noauth.policy_data
}

R代码片段（API服务） #

runPipeline = function(run_container){
  body = as.character(run_container)
  if (body == "run") {  
    date = as.character(dbGetQuery(con, q)$`f0_`)
    if (date == "2017-01-01") {
      load2BQinitial(data = reformatData(data=cleanData(df=getData(date=date))))
    }
    if (date != "2017-01-01") {
      load2BQincremental(data = reformatData(data=cleanData(df=getData(date=date))))
    }
    return((paste0("Running the pipeline starting from ", date)))
  }
  else {
    return((paste0("Something went wrong, please make sure GET request is sent correctly.")))
  }
}


# -- Create API endpoint to receive & feed new data to model -- #  

newBeakr() %>% 
  httpGET(path = "/launch", decorate(runPipeline)) %>%        # Respond to GET requests at the "/launch" route
  handleErrors() %>%                                          # Handle any errors with a JSON response
  listen(host = "0.0.0.0", port = 8080)                       # Start the server on port 8080

排查与解决步骤 #

1. 补全请求路径 #

你的R代码中API端点是/launch，但当前Cloud Scheduler和curl请求只指向Cloud Run服务根域名，导致请求找不到对应端点。修改Cloud Scheduler的Terraform配置：

http_target {
  http_method = "GET"
  uri         = "${google_cloud_run_service.default.status[0].url}/launch?run_container=run"
  # 其他配置保持不变
}

2. 修复GET参数传递逻辑 #

runPipeline函数依赖run_container参数，但decorate(runPipeline)无法自动从GET请求中提取查询参数。调整R代码的端点处理逻辑：

httpGET(path = "/launch", function(req, res, err) {
  run_container <- req$queryParams$run_container
  result <- runPipeline(run_container)
  sendText(res, result)
})

3. 完善Cloud Run端口配置 #

Cloud Run默认使用HTTP/2协议与容器通信，需在容器端口配置中添加h2c标识：

ports {
  name           = "h2c"
  container_port = 8080
}

4. 确认服务账号权限 #

虽然已开放allUsers调用权限，但Cloud Scheduler使用的服务账号仍需单独授予roles/run.invoker角色：

data "google_iam_policy" "cloud_scheduler_invoker" {
  binding {
    role = "roles/run.invoker"
    members = [
      "allUsers",
      "serviceAccount:${google_service_account.default.email}"
    ]
  }
}

resource "google_cloud_run_service_iam_policy" "noauth" {
  location    = google_cloud_run_service.default.location
  project     = google_cloud_run_service.default.project
  service     = google_cloud_run_service.default.name
  policy_data = data.google_iam_policy.cloud_scheduler_invoker.policy_data
}

5. 检查容器启动状态 #

通过Cloud Console查看Cloud Run实例的启动日志，确认R应用是否成功绑定8080端口，是否存在依赖缺失、数据库连接失败等启动错误。

6. 调整请求超时设置 #

当前curl请求延迟达2.7秒，若流水线执行时间较长，需延长Cloud Run的请求超时：

template {
  spec {
    containers {
      # 其他配置不变
    }
    timeout_seconds = 120 # 根据实际执行时间调整
  }
}

内容的提问来源于stack exchange，提问作者Keith Cozart