GCP App Engine强制HTTPS检查脚本无法获取app.yaml问题排查

问题背景 #

需要验证GCP所有项目中App Engine版本的app.yaml是否配置了强制HTTPS规则，要求的标准配置如下：

handlers:
- url: /.*
  secure: always
  redirect_http_response_code: 301
  script: auto

编写的Python脚本可正常获取所有项目的App Engine版本ID，但始终无法读取对应版本的app.yaml内容，返回"No app.yaml found"。原脚本代码如下：

import subprocess
import csv
import re

# Get a list of all Google Cloud projects
projects = subprocess.run(["gcloud", "projects", "list", "--format=value(projectId)"], capture_output=True, text=True)
project_ids = projects.stdout.splitlines()

results = []

for project_id in project_ids:
    # Get versions for App Engine services within each project
    app_versions_command = f"gcloud app versions list --format='table(version.id)' --project={project_id} --service=default"
    app_versions_output = subprocess.run(app_versions_command, shell=True, capture_output=True, text=True)

    if app_versions_output.returncode == 0:
        versions = app_versions_output.stdout.splitlines()[1:]  # Skip header
        for version in versions:
            version = version.strip()
            # Get app.yaml content for each version
            app_yaml_command = f"gcloud app versions describe {version} --project={project_id} --service=default --format='get(config.appYaml)'"
            app_yaml_content = subprocess.run(app_yaml_command, shell=True, capture_output=True, text=True)

            if app_yaml_content.returncode == 0:
                yaml_content = app_yaml_content.stdout
                # Check if the app.yaml file contains the specific configuration
                if re.search(r'handlers:\s*- url: /.*\s*  secure: always\s*  redirect_http_response_code: 301\s*  script: auto', yaml_content):
                    results.append({"Project ID": project_id, "Version ID": version, "Secure Connection": "Enforced"})
                else:
                    results.append({"Project ID": project_id, "Version ID": version, "Secure Connection": "Not Enforced"})
            else:
                results.append({"Project ID": project_id, "Version ID": version, "Secure Connection": "No app.yaml found"})
    else:
        results.append({"Project ID": project_id, "Version ID": "N/A", "Secure Connection": "Error fetching versions"})

# Export results to a CSV file
with open('secure_connections_all_versions.csv', 'w', newline='') as csvfile:
    fieldnames = ['Project ID', 'Version ID', 'Secure Connection']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

    writer.writeheader()
    writer.writerows(results)

print("Results exported to secure_connections_all_versions.csv")

排查与解决 #

核心问题点 #

1. CLI格式错误：原脚本中--format='get(config.appYaml)'的写法无效，GCP CLI并未将appYaml作为config下的直接可提取字段，实际需要获取完整的配置结构后解析。
2. 正则匹配过于严苛：原正则要求配置的缩进、换行完全一致，实际部署的app.yaml可能存在格式差异，导致误判未配置。
3. 权限不足：运行脚本的账号可能缺少appengine.versions.get权限，需确保账号拥有App Engine Viewer角色。

修改后的脚本 #

使用PyYAML解析配置，替换正则匹配，调整CLI命令以正确获取配置：

import subprocess
import csv
import yaml
from typing import Dict, List

def get_gcp_projects() -> List[str]:
    """获取所有GCP项目ID"""
    result = subprocess.run(
        ["gcloud", "projects", "list", "--format=value(projectId)"],
        capture_output=True,
        text=True,
        check=False
    )
    return [p.strip() for p in result.stdout.splitlines() if p.strip()]

def get_app_versions(project_id: str) -> List[str]:
    """获取指定项目默认服务的App Engine版本ID"""
    result = subprocess.run(
        ["gcloud", "app", "versions", "list", "--project", project_id, "--service=default", "--format=value(version.id)"],
        capture_output=True,
        text=True,
        check=False
    )
    if result.returncode != 0:
        return []
    return [v.strip() for v in result.stdout.splitlines() if v.strip()]

def check_secure_config(project_id: str, version_id: str) -> str:
    """检查指定版本的强制HTTPS配置"""
    result = subprocess.run(
        ["gcloud", "app", "versions", "describe", version_id, "--project", project_id, "--service=default", "--format=yaml(config)"],
        capture_output=True,
        text=True,
        check=False
    )
    if result.returncode != 0:
        return "Error fetching config"
    
    try:
        config = yaml.safe_load(result.stdout)
        if not config or 'handlers' not in config:
            return "No handlers configured"
        
        for handler in config['handlers']:
            if handler.get('url') == '/.*' and handler.get('secure') == 'always' and handler.get('redirect_http_response_code') == 301:
                return "Enforced"
        return "Not Enforced"
    except yaml.YAMLError:
        return "Invalid config format"

def main():
    results = []
    projects = get_gcp_projects()
    
    for project in projects:
        versions = get_app_versions(project)
        if not versions:
            results.append({
                "Project ID": project,
                "Version ID": "N/A",
                "Secure Connection": "No versions found or error fetching"
            })
            continue
        
        for version in versions:
            status = check_secure_config(project, version)
            results.append({
                "Project ID": project,
                "Version ID": version,
                "Secure Connection": status
            })
    
    # 导出到CSV
    with open('secure_connections_all_versions.csv', 'w', newline='', encoding='utf-8') as csvfile:
        fieldnames = ['Project ID', 'Version ID', 'Secure Connection']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(results)
    
    print("Results exported to secure_connections_all_versions.csv")

if __name__ == "__main__":
    main()

使用说明 #

1. 确保已安装PyYAML：pip install pyyaml
2. 确保运行脚本的账号已通过gcloud auth login认证，且拥有足够权限
3. 脚本会自动遍历所有项目的默认服务版本，检查是否配置了符合要求的强制HTTPS规则

内容的提问来源于stack exchange，提问作者Touhid Alam