Terraform创建AWS Athena Iceberg表时parameters参数未生效

使用Terraform创建AWS Athena Iceberg表时，在aws_glue_catalog_table资源块的顶层parameters参数中添加的键值对未被填充到最终创建的Iceberg表中。执行Show create Table查询后，仅能看到默认的table_type='iceberg'属性，自定义的vacuum_max_snapshot_age_seconds、write_compression等参数均缺失。

原Terraform配置 #

resource "aws_glue_catalog_database" "athena_fluent-inventory_db" {
  name = "fluent_inventory_database"
}

resource "aws_glue_catalog_table" "fluent_inventory_snapshot_iceberg_table" {
  name          = "fluent_inventory_snapshot_iceberg_table"
  database_name = aws_glue_catalog_database.athena_fluent-inventory_db.name

  table_type = "EXTERNAL_TABLE"

  open_table_format_input {
    iceberg_input {
      metadata_operation = "CREATE"
      version            = "2"
    }
  }

  storage_descriptor {
    location      = "s3://${var.fluentInventorySnapshotIcebergStorageBucketName}/"
    input_format  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
    output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"

    ser_de_info {
      serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
    }
    columns {
      name = "id"
      type = "string"
    }
    columns {
      name = "status"
      type = "string"
    }
    columns {
      name = "quantity"
      type = "bigint"
    }
    columns {
      name = "group_reference"
      type = "string"
    }
    columns {
      name = "virtual_position_type"
      type = "string"
    }
    columns {
      name = "created_on"
      type = "timestamp"
    }
    columns {
      name = "updated_on"
      type = "timestamp"
    }
    columns {
      name = "product_reference"
      type = "string"
    }
    columns {
      name = "catalogue_id"
      type = "string"
    }
    columns {
      name = "external_reference"
      type = "string"
    }
  }

  parameters = {
    "vacuum_max_snapshot_age_seconds" = "2592000"
    "write_compression"               = "snappy"
    "format"                          = "parquet"
  }
}

查询返回的创建语句 #

CREATE TABLE fluent_inventory_database.fluent_inventory_snapshot_iceberg_table (
  id string COMMENT '',
  status string COMMENT '',
  quantity bigint COMMENT '',
  group_reference string COMMENT '',
  virtual_position_type string COMMENT '',
  created_on timestamp COMMENT '',
  updated_on timestamp COMMENT '',
  product_reference string COMMENT '',
  catalogue_id string COMMENT '',
  external_reference string COMMENT '')
LOCATION 's3://fluent-inventory-snapshot-iceberg-storage-default'
TBLPROPERTIES (
  'table_type'='iceberg'
);

对于Iceberg格式的表，Terraform的aws_glue_catalog_table资源要求自定义参数必须放在open_table_format_input.iceberg_input.parameters块中，而非顶层的parameters字段。顶层parameters仅适用于普通Glue表，Iceberg表的属性需通过专属参数块传递。

修改后的Terraform配置 #

resource "aws_glue_catalog_database" "athena_fluent-inventory_db" {
  name = "fluent_inventory_database"
}

resource "aws_glue_catalog_table" "fluent_inventory_snapshot_iceberg_table" {
  name          = "fluent_inventory_snapshot_iceberg_table"
  database_name = aws_glue_catalog_database.athena_fluent-inventory_db.name

  table_type = "EXTERNAL_TABLE"

  open_table_format_input {
    iceberg_input {
      metadata_operation = "CREATE"
      version            = "2"
      # 将自定义参数移至此处
      parameters = {
        "vacuum_max_snapshot_age_seconds" = "2592000"
        "write_compression"               = "snappy"
        "format"                          = "parquet"
      }
    }
  }

  storage_descriptor {
    location      = "s3://${var.fluentInventorySnapshotIcebergStorageBucketName}/"
    input_format  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
    output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"

    ser_de_info {
      serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
    }
    columns {
      name = "id"
      type = "string"
    }
    columns {
      name = "status"
      type = "string"
    }
    columns {
      name = "quantity"
      type = "bigint"
    }
    columns {
      name = "group_reference"
      type = "string"
    }
    columns {
      name = "virtual_position_type"
      type = "string"
    }
    columns {
      name = "created_on"
      type = "timestamp"
    }
    columns {
      name = "updated_on"
      type = "timestamp"
    }
    columns {
      name = "product_reference"
      type = "string"
    }
    columns {
      name = "catalogue_id"
      type = "string"
    }
    columns {
      name = "external_reference"
      type = "string"
    }
  }

  # 移除顶层的parameters字段
}

说明 #

修改后重新执行Terraform部署，自定义参数会被正确写入Iceberg表的TBLPROPERTIES中，再次执行Show create Table即可看到这些参数。

内容的提问来源于stack exchange，提问作者sebin vincent