跨AWS账号迁移Terraform部署的Jaeger-ES服务连接失败求助

安全阿岳

2026-8-24

问题：ECS Fargate上的Jaeger服务无法连接新AWS账号中的Elasticsearch

我通过Terraform在ECS Fargate上部署了Jaeger服务，该服务在原AWS账号运行正常，但切换至另一AWS账号的访问密钥后，Elasticsearch域与ECS任务已创建完成，但ECS任务无法连接Elasticsearch，报错信息如下：

{"level":"fatal","ts":1658741608.5497034,"caller":"command-line-arguments/main.go:103","msg":"Failed to init storage factory","error":"failed to create primary Elasticsearch client: health check timeout: no Elasticsearch node available","stacktrace":"main.main.func1\n\tcommand-line-arguments/main.go:103\ngithub.com/spf13/cobra.(*Command).execute\n\tgithub.com/spf13/cobra@v1.2.1/command.go:856\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\tgithub.com/spf13/cobra@v1.2.1/command.go:974\ngithub.com/spf13/cobra.(*Command).Execute\n\tgithub.com/spf13/cobra@v1.2.1/command.go:902\nmain.main\n\tcommand-line-arguments/main.go:216\nruntime.main\n\truntime/proc.go:225"}

我的Terraform代码

resource "aws_security_group" "es" {
  name        = "jaeger-es-sg"
  description = "Allow inbound traffic to ElasticSearch from VPC CIDR"
  vpc_id      = aws_vpc.vpc.id

  ingress {
    from_port        = 14269
    to_port          = 14269
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 14268
    to_port          = 14268
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 6832
    to_port          = 6832
    protocol         = "udp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 6831
    to_port          = 6831
    protocol         = "udp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 5775
    to_port          = 5775
    protocol         = "udp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 14250
    to_port          = 14250
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 16685
    to_port          = 16685
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 5778
    to_port          = 5778
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 16686
    to_port          = 16686
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 9411
    to_port          = 9411
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 80
    to_port          = 80
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  ingress {
    from_port        = 443
    to_port          = 443
    protocol         = "tcp"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
}

resource "aws_iam_service_linked_role" "es" {
  aws_service_name = "es.amazonaws.com"
}

resource "aws_elasticsearch_domain" "es" {
  domain_name           = "jaeger-elasticsearch"
  elasticsearch_version = "7.10"

  cluster_config {
    instance_count         = 2
    instance_type          = "t3.small.elasticsearch"
    zone_awareness_enabled = true

    zone_awareness_config {
      availability_zone_count = 2
    }
  }

  vpc_options {
    subnet_ids = [
      aws_subnet.subnet-a.id,
      aws_subnet.subnet-b.id
    ]

    security_group_ids = [
      aws_security_group.es.id
    ]
  }

  ebs_options {
    ebs_enabled = true
    volume_size = 10
  }

  access_policies = <<CONFIG
{
  "Version": "2012-10-17",
  "Statement": [
      {
          "Action": "es:*",
          "Principal": "*",
          "Effect": "Allow",
          "Resource": "arn:aws:es:eu-central-1:-----------:domain/jaeger-elasticsearch"
      }
  ]
}
  CONFIG

  snapshot_options {
    automated_snapshot_start_hour = 23
  }

  tags = {
    Domain = "jaeger-elasticsearch"
  }
}
resource "aws_ecs_cluster" "jaeger-cluster" {
  name = "jaeger-cluster"
  tags = {
    Name = "jaeger-cluster"
  }
}

resource "aws_ecs_task_definition" "jaegerTD" {
  family                   = "jaegerTD"
  requires_compatibilities = ["FARGATE"]
  network_mode             = "awsvpc"
  memory                   = 2048
  cpu                      = 1024
  execution_role_arn       = data.aws_iam_role.ecs_task_execution_role.arn
  task_role_arn            = data.aws_iam_role.ecs_task_execution_role.arn


  container_definitions = <<TASK_DEFINITION
[
  {
      "dnsSearchDomains": [],
      "environmentFiles": null,
      "logConfiguration": {
        "logDriver": "awslogs",
        "secretOptions": null,
        "options": {
          "awslogs-group": "/ecs/jaeger",
          "awslogs-region": "eu-central-1",
          "awslogs-create-group": "true",
          "awslogs-stream-prefix": "ecs"
        }
      },
      "entryPoint": [],
      "portMappings": [
        {
          "hostPort": 14269,
          "protocol": "tcp",
          "containerPort": 14269
        },
        {
          "hostPort": 14268,
          "protocol": "tcp",
          "containerPort": 14268
        },
        {
          "hostPort": 6832,
          "protocol": "udp",
          "containerPort": 6832
        },
        {
          "hostPort": 6831,
          "protocol": "udp",
          "containerPort": 6831
        },
        {
          "hostPort": 5775,
          "protocol": "udp",
          "containerPort": 5775
        },
        {
          "hostPort": 14250,
          "protocol": "tcp",
          "containerPort": 14250
        },
        {
          "hostPort": 16685,
          "protocol": "tcp",
          "containerPort": 16685
        },
        {
          "hostPort": 5778,
          "protocol": "tcp",
          "containerPort": 5778
        },
        {
          "hostPort": 16686,
          "protocol": "tcp",
          "containerPort": 16686
        },
        {
          "hostPort": 9411,
          "protocol": "tcp",
          "containerPort": 9411
        }
      ],
      "command": [
        "--collector.zipkin.host-port",
        "9411"
      ],
      "linuxParameters": null,
      "cpu": 1024,
      "environment": [
        {
          "name": "ES_SERVER_URLS",
          "value": "https://vpc-jaeger-elasticsearch----------------------.eu-central-1.es.amazonaws.com/"
        },
        {
          "name": "SPAN_STORAGE_TYPE",
          "value": "elasticsearch"
        }
      ],
      "resourceRequirements": null,
      "ulimits": null,
      "dnsServers": [],
      "mountPoints": [],
      "workingDirectory": null,
      "secrets": null,
      "dockerSecurityOptions": [],
      "memory": 1024,
      "memoryReservation": null,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": "jaegertracing/all-in-one:1.25.0",
      "startTimeout": null,
      "firelensConfiguration": null,
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": [],
      "hostname": null,
      "extraHosts": null,
      "pseudoTerminal": null,
      "user": null,
      "readonlyRootFilesystem": null,
      "dockerLabels": null,
      "systemControls": [],
      "privileged": null,
      "name": "container-name"
    }
]
TASK_DEFINITION

}
resource "aws_ecs_service" "jaeger-service" {
  name            = "jaeger-service"
  cluster         = aws_ecs_cluster.jaeger-cluster.id
  task_definition = aws_ecs_task_definition.jaegerTD.id
  desired_count   = 1
  depends_on = [
    aws_ecs_cluster.jaeger-cluster,
    aws_ecs_task_definition.jaegerTD
  ]

  launch_type                        = "FARGATE"
  deployment_minimum_healthy_percent = 0
  deployment_maximum_percent         = 100

  network_configuration {
    subnets          = [aws_subnet.subnet-a.id, aws_subnet.subnet-b.id]
    security_groups  = [aws_security_group.es.id]
    assign_public_ip = true
  }
  service_registries {
    registry_arn   = aws_service_discovery_service.jaeger.arn
    container_name = "container-name"
  }
}
resource "aws_service_discovery_private_dns_namespace" "jaeger-namespace" {
  name        = "trace.com"
  description = "service discovery for Jaeger"
  vpc         = aws_vpc.vpc.id
}

resource "aws_service_discovery_service" "jaeger" {
  name = "jaeger"

  dns_config {
    namespace_id = aws_service_discovery_private_dns_namespace.jaeger-namespace.id

    dns_records {
      ttl  = 10
      type = "A"
    }
    routing_policy = "MULTIVALUE"
  }

  health_check_custom_config {
    failure_threshold = 1
  }
}

排查与修复步骤

修正Elasticsearch访问策略中的账号ID：aws_elasticsearch_domain.es的access_policies里的ARN包含硬编码的原账号ID（-----------），替换为当前账号ID，或通过Terraform动态获取：

data "aws_caller_identity" "current" {}

access_policies = <<CONFIG
{
  "Version": "2012-10-17",
  "Statement": [
      {
          "Action": "es:*",
          "Principal": "*",
          "Effect": "Allow",
          "Resource": "arn:aws:es:eu-central-1:${data.aws_caller_identity.current.account_id}:domain/jaeger-elasticsearch"
      }
  ]
}
CONFIG

动态生成ES_SERVER_URLS：任务定义中ES_SERVER_URLS硬编码了原账号的ES域名，改为引用Terraform资源属性动态生成：

environment = [
  {
    "name": "ES_SERVER_URLS",
    "value": "https://${aws_elasticsearch_domain.es.endpoint}/"
  },
  {
    "name": "SPAN_STORAGE_TYPE",
    "value": "elasticsearch"
  }
]

验证VPC与子网连通性：确认ECS任务和Elasticsearch处于同一个VPC，子网路由表允许VPC内部流量。关闭Fargate任务的assign_public_ip = true，确保任务使用内网访问ES，避免公网访问限制。
检查Elasticsearch域状态：在AWS控制台确认Elasticsearch域状态为Active，等待集群完全初始化后再启动ECS任务。
验证IAM权限：确保ECS任务角色具备访问Elasticsearch的权限，虽然访问策略允许所有主体，但若存在VPC端点策略，需补充对应权限。

内容的提问来源于stack exchange，提问作者Meriç Özkayagan

跨AWS账号迁移Terraform部署的Jaeger-ES服务连接失败求助

我的Terraform代码 #

排查与修复步骤 #

我的Terraform代码

排查与修复步骤