Terraform在AWS对等VPC中反复创建删除路由表问题排查

问题现象 #

同一AWS账号同区域下部署两个VPC并创建对等连接，首次执行terraform apply可正常创建VPC peering及对应路由，EC2实例能互通。但后续执行出现循环：

• 第二次执行：Terraform会删除路由表中的对等路由
• 第三次执行：Terraform又重新创建该对等路由

原Terraform代码 #

provider "aws" {
  region                = "..."
  access_key            = "..."
  secret_key            = "..."
}

resource "aws_vpc" "vpc1" {
  cidr_block = "10.0.0.0/16"
}

resource "aws_vpc" "vpc2" {
  cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "public1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.1.0/24"
}

resource "aws_subnet" "private1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.2.0/24"
}

resource "aws_subnet" "public2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.1.0/24"
}

resource "aws_subnet" "private2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.2.0/24"
}

resource "aws_internet_gateway" "igw1" {
  vpc_id = aws_vpc.vpc1.id
}

resource "aws_internet_gateway" "igw2" {
  vpc_id = aws_vpc.vpc2.id
}

resource "aws_route_table" "public_route_table1" {
  vpc_id = aws_vpc.vpc1.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw1.id
  }
}

resource "aws_route_table" "public_route_table2" {
  vpc_id = aws_vpc.vpc2.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw2.id
  }
}

resource "aws_route_table_association" "public1_rta" {
  subnet_id      = aws_subnet.public1.id
  route_table_id = aws_route_table.public_route_table1.id
}

resource "aws_route_table_association" "public2_rta" {
  subnet_id      = aws_subnet.public2.id
  route_table_id = aws_route_table.public_route_table2.id
}

resource "aws_vpc_peering_connection" "peer" {
  vpc_id      = aws_vpc.vpc1.id
  peer_vpc_id = aws_vpc.vpc2.id
  auto_accept = true
}

resource "aws_route" "peer1_to_2" {
  route_table_id            = aws_route_table.public_route_table1.id
  destination_cidr_block    = aws_vpc.vpc2.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route" "peer2_to_1" {
  route_table_id            = aws_route_table.public_route_table2.id
  destination_cidr_block    = aws_vpc.vpc1.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

第二次执行terraform plan输出 #

# aws_route_table.public_route_table1 will be updated in-place
  ~ resource "aws_route_table" "public_route_table1" {
        id               = "rtb-0ad43422d1bc73f82"
      ~ route            = [
          - {
              - cidr_block                 = "0.0.0.0/0"
              - gateway_id                 = "igw-06ef35a787e283282"
                # (11 unchanged attributes hidden)
            },
          - {
              - cidr_block                 = "10.1.0.0/16"
              - vpc_peering_connection_id  = "pcx-0afbf2e62bf9a43d2"
                # (11 unchanged attributes hidden)
            },
          + {
              + cidr_block = "0.0.0.0/0"
              + gateway_id = "igw-06ef35a787e283282"
            },
        ]
        tags             = {}
        # (5 unchanged attributes hidden)
    }

  # aws_route_table.public_route_table2 will be updated in-place
  ~ resource "aws_route_table" "public_route_table2" {
        id               = "rtb-0f282179701ca2405"
      ~ route            = [
          - {
              - cidr_block                 = "0.0.0.0/0"
              - gateway_id                 = "igw-015a5ba053949c7cf"
                # (11 unchanged attributes hidden)
            },
          - {
              - cidr_block                 = "10.0.0.0/16"
              - vpc_peering_connection_id  = "pcx-0afbf2e62bf9a43d2"
                # (11 unchanged attributes hidden)
            },
          + {
              + cidr_block = "0.0.0.0/0"
              + gateway_id = "igw-015a5ba053949c7cf"
            },
        ]
        tags             = {}
        # (5 unchanged attributes hidden)
    }

Plan: 0 to add, 2 to change, 0 to destroy.

第三次执行terraform plan输出 #

# aws_route.peer1_to_2 will be created
  + resource "aws_route" "peer1_to_2" {
      + destination_cidr_block    = "10.1.0.0/16"
      + id                        = (known after apply)
      + instance_id               = (known after apply)
      + instance_owner_id         = (known after apply)
      + network_interface_id      = (known after apply)
      + origin                    = (known after apply)
      + route_table_id            = "rtb-0ad43422d1bc73f82"
      + state                     = (known after apply)
      + vpc_peering_connection_id = "pcx-0afbf2e62bf9a43d2"
    }

  # aws_route.peer2_to_1 will be created
  + resource "aws_route" "peer2_to_1" {
      + destination_cidr_block    = "10.0.0.0/16"
      + id                        = (known after apply)
      + instance_id               = (known after apply)
      + instance_owner_id         = (known after apply)
      + network_interface_id      = (known after apply)
      + origin                    = (known after apply)
      + route_table_id            = "rtb-0f282179701ca2405"
      + state                     = (known after apply)
      + vpc_peering_connection_id = "pcx-0afbf2e62bf9a43d2"
    }

Plan: 2 to add, 0 to change, 0 to destroy.

问题原因 #

核心是同时混用了aws_route_table内嵌的route块和独立的aws_route资源：

• aws_route_table资源会严格管理其内嵌route块定义的所有路由，当它检测到路由表中存在未在块内定义的路由（由aws_route创建的对等路由），会将这些路由判定为“漂移资源”并执行删除操作。
• 独立的aws_route资源会监控自身定义的路由状态，当发现路由被删除后，会触发重新创建操作，最终导致循环。

解决方案 #

两种方式二选一，统一路由的管理方式：

方案1：将所有路由整合到aws_route_table内嵌route块中 #

删除独立的aws_route资源，将对等路由直接添加到aws_route_table的route块内：

resource "aws_route_table" "public_route_table1" {
  vpc_id = aws_vpc.vpc1.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw1.id
  }

  route {
    cidr_block                 = aws_vpc.vpc2.cidr_block
    vpc_peering_connection_id  = aws_vpc_peering_connection.peer.id
  }
}

resource "aws_route_table" "public_route_table2" {
  vpc_id = aws_vpc.vpc2.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw2.id
  }

  route {
    cidr_block                 = aws_vpc.vpc1.cidr_block
    vpc_peering_connection_id  = aws_vpc_peering_connection.peer.id
  }
}

# 删除原有的aws_route.peer1_to_2和aws_route.peer2_to_1资源

方案2：全部使用独立aws_route资源管理路由 #

移除aws_route_table中的内嵌route块，将默认路由也改为独立的aws_route资源：

resource "aws_route_table" "public_route_table1" {
  vpc_id = aws_vpc.vpc1.id
  # 移除内嵌的route块
}

resource "aws_route_table" "public_route_table2" {
  vpc_id = aws_vpc.vpc2.id
  # 移除内嵌的route块
}

# 添加默认路由的独立aws_route资源
resource "aws_route" "default1" {
  route_table_id = aws_route_table.public_route_table1.id
  cidr_block     = "0.0.0.0/0"
  gateway_id     = aws_internet_gateway.igw1.id
}

resource "aws_route" "default2" {
  route_table_id = aws_route_table.public_route_table2.id
  cidr_block     = "0.0.0.0/0"
  gateway_id     = aws_internet_gateway.igw2.id
}

# 保留原有的对等路由aws_route资源
resource "aws_route" "peer1_to_2" {
  route_table_id            = aws_route_table.public_route_table1.id
  destination_cidr_block    = aws_vpc.vpc2.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route" "peer2_to_1" {
  route_table_id            = aws_route_table.public_route_table2.id
  destination_cidr_block    = aws_vpc.vpc1.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

注意事项 #

修改后执行terraform apply前，建议先执行terraform plan确认变更符合预期，避免意外删除资源。

内容的提问来源于stack exchange，提问作者Luciano Sarra