使用Docker Swarm部署Cassandra集群失败：容器互连异常

安全阿岳

2026-8-15

Docker Swarm部署Cassandra集群节点连接失败排查

问题现象

使用Docker Swarm配置Cassandra集群时，容器反复停止重启，节点间无法建立连接，集群状态异常。执行nodetool describecluster输出：

root@f6d8d694f8d4:/# nodetool describecluster
Cluster Information:
    Name: Test Cluster
    Snitch: org.apache.cassandra.locator.SimpleSnitch
    DynamicEndPointSnitch: enabled
    Partitioner: org.apache.cassandra.dht.Murmur3Partitioner
    Schema versions:
        UNREACHABLE: [10.0.8.2, 10.0.2.2]

日志片段显示连接超时：

io.netty.channel.ConnectTimeoutException: connection timed out: /10.0.2.2:7000
    at io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe$2.run(AbstractEpollChannel.java:576)
    at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98)
    at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170)
    at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
    at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
    at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:384)
    at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
    at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
    at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.base/java.lang.Thread.run(Unknown Source)
WARN  [OptionalTasks:1] 2022-10-19 20:29:55,349 CassandraRoleManager.java:344 - CassandraRoleManager skipped default role setup: some nodes were not ready

相关配置文件

Dockerfile

FROM cassandra:4.0.6
RUN apt -y update && apt -y install net-tools && apt install iputils-ping

存在问题的Docker Compose文件

version: '3.2'

services:
  cassandra0:
    image: my_cassandra:latest
    volumes:
      - ./cassandra/0:/var/lib/cassandra 
    networks:
      - cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra0"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "512M"
    ports:
      - 7000
      - 9072
      - 7199
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
  cassandra1:
    image: my_cassandra:latest
    volumes:
      - ./cassandra/1:/var/lib/cassandra
    networks:
      - cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra1"
      CASSANDRA_SEEDS: "cassandra0"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "512M"
    ports:
      - 7000
      - 9072
      - 7199
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
  cassandra2:
    image: my_cassandra:latest
    volumes:
      - ./cassandra/2:/var/lib/cassandra
    networks:
      - cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra2"
      CASSANDRA_SEEDS: "cassandra0"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "512M"
    ports:
      - 7000
      - 9072
      - 7199
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
  cassandra3:
    image: my_cassandra:latest
    volumes:
      - ./cassandra/3:/var/lib/cassandra
    networks:
      - cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra3"
      CASSANDRA_SEEDS: "cassandra0"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "512M"
    ports:
      - 7000
      - 9072
      - 7199
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
  cassandra4:
    image: my_cassandra:latest
    volumes:
      - ./cassandra/4:/var/lib/cassandra
    networks:
      - cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra4"
      CASSANDRA_SEEDS: "cassandra0"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "512M"
    ports:
      - 7000
      - 9072
      - 7199
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
networks:
  cassandra:
          external: true

Overlay网络信息

...
"Name": "cassandra",
        "Id": "w4ooyvfgyuslnlhl0e9tmyfw2",
        "Created": "2022-10-19T22:37:14.60262042+02:00",
        "Scope": "swarm",
        "Driver": "overlay",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "10.0.10.0/24",
                    "Gateway": "10.0.10.1"
                }
            ]
        }
...

可正常运行的Docker Compose文件

version: "3.2"
  
services:  

  ################################################################
  # The Casandra cluster 
  #   - cassandra-node1
  ################################################################        
  cassandra-001:
    image: my_cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra-001"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "1024M"
    volumes:
      - ./volumes/001:/var/lib/cassandra
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
    networks:
      - cluster_net

  ################################################################
  # The Casandra cluster 
  #   - cassandra-node2
  ################################################################        
  cassandra-002:
    image: my_cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra-002"
      CASSANDRA_SEEDS: "cassandra-001"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "1024M"
    volumes:
      - ./volumes/002:/var/lib/cassandra
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
    networks:
      - cluster_net

  ################################################################
  # The Casandra cluster 
  #   - cassandra-node3
  ################################################################        
  cassandra-003:
    image: my_cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra-003"
      CASSANDRA_SEEDS: "cassandra-001"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "1024M"
    volumes:
      - ./volumes/003:/var/lib/cassandra
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
    networks:
      - cluster_net

  ################################################################
  # The Casandra cluster 
  #   - cassandra-node4
  ################################################################        
  cassandra-004:
    image: my_cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra-004"
      CASSANDRA_SEEDS: "cassandra-001"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "1024M"
    volumes:
      - ./volumes/004:/var/lib/cassandra
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
    networks:
      - cluster_net

  ################################################################
  # The Casandra cluster 
  #   - cassandra-node5
  ################################################################        
  cassandra-005:
    image: my_cassandra
    environment:
      CASSANDRA_BROADCAST_ADDRESS: "cassandra-005"
      CASSANDRA_SEEDS: "cassandra-001"
      HEAP_NEWSIZE: "128M"
      MAX_HEAP_SIZE: "1024M"
    volumes:
      - ./volumes/005:/var/lib/cassandra
    deploy:
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
    networks:
      - cluster_net

networks:
  cluster_net:
    external:
      name: cassandra

问题排查与解决方案

对比两个Compose文件，核心差异及修复方案如下：

Java堆内存不足
有问题的配置中MAX_HEAP_SIZE设为512M，远低于正常配置的1024M。Cassandra对内存需求较高，过小的堆内存会导致节点初始化失败、OOM或无法响应连接请求。
修复：将所有节点的MAX_HEAP_SIZE调整为1024M。
不必要的端口暴露干扰
Docker Swarm的overlay网络内，服务间通信无需暴露端口。随机映射端口可能导致节点尝试通过宿主机端口通信，而非直接使用overlay网络，引发连接超时。
修复：移除所有节点的ports配置项。
遗留损坏数据干扰
之前启动失败的节点可能留下损坏的数据卷，导致重启后无法正常加入集群。
修复：清理所有节点的本地数据卷（./cassandra/0至./cassandra/4）。

节点启动顺序混乱
Swarm默认同时启动所有节点，seed节点未就绪时其他节点就尝试连接，导致失败。
修复：为seed节点添加健康检查，其他节点依赖seed节点健康状态启动：

cassandra0:
  # ...其他配置
  healthcheck:
    test: ["CMD", "nodetool", "status"]
    interval: 30s
    timeout: 10s
    retries: 5
cassandra1:
  # ...其他配置
  depends_on:
    cassandra0:
      condition: service_healthy

最终修复步骤：

修改有问题的Compose文件，调整堆内存并移除ports配置
清理本地数据卷
重新部署集群：docker stack deploy -c docker-compose.yml cassandra-cluster

内容的提问来源于stack exchange，提问作者fcracker79

使用Docker Swarm部署Cassandra集群失败：容器互连异常

问题现象 #

相关配置文件 #

Dockerfile #

存在问题的Docker Compose文件 #

Overlay网络信息 #

可正常运行的Docker Compose文件 #

问题排查与解决方案 #