基于Docker启用双VM搭建Kafka多节点集群的技术问询

以下是针对两台Docker虚拟机搭建高可用Kafka集群的实操方案，支持后续扩展更多节点：

前提说明 #

假设两台VM的局域网IP分别为：

• VM1: 192.168.1.10
• VM2: 192.168.1.11
  两台机器均已安装Docker和Docker Compose，且防火墙开放对应端口（后续配置中涉及的9092/9093/9094或2181/2888/3888）。

推荐方案：KRaft模式（无ZooKeeper） #

Kafka 2.8+官方推荐使用KRaft替代ZooKeeper，简化架构且提升稳定性。

1. 生成全局集群ID #

在任意一台VM执行命令生成唯一集群ID，所有节点共用：

docker run --rm apache/kafka:latest kafka-storage.sh random-uuid

记录输出的ID（如abc123-def456-ghi789）。

2. VM1的docker-compose.yml配置 #

version: '3.8'
services:
  kafka1:
    image: apache/kafka:latest
    container_name: kafka1
    ports:
      - "9092:9092"  # 容器内部通信端口
      - "9093:9093"  # 外部客户端/跨节点通信端口
      - "9094:9094"  # KRaft控制器端口
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENERS: INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093,CONTROLLER://0.0.0.0:9094
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:9092,EXTERNAL://192.168.1.10:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka1:9094,2@192.168.1.11:9094
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2
      KAFKA_STORAGE_DIRECTORIES: /tmp/kafka-storage
    volumes:
      - ./kafka1-storage:/tmp/kafka-storage
    command: >
      bash -c "kafka-storage.sh format -t abc123-def456-ghi789 -c /opt/kafka/config/kraft/server.properties && exec kafka-server-start.sh /opt/kafka/config/kraft/server.properties"

3. VM2的docker-compose.yml配置 #

version: '3.8'
services:
  kafka2:
    image: apache/kafka:latest
    container_name: kafka2
    ports:
      - "9092:9092"
      - "9093:9093"
      - "9094:9094"
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_LISTENERS: INTERNAL://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093,CONTROLLER://0.0.0.0:9094
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:9092,EXTERNAL://192.168.1.11:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@192.168.1.10:9094,2@kafka2:9094
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2
      KAFKA_STORAGE_DIRECTORIES: /tmp/kafka-storage
    volumes:
      - ./kafka2-storage:/tmp/kafka-storage
    command: >
      bash -c "kafka-storage.sh format -t abc123-def456-ghi789 -c /opt/kafka/config/kraft/server.properties && exec kafka-server-start.sh /opt/kafka/config/kraft/server.properties"

4. 启动与验证 #

• 分别在两台VM执行：

  docker-compose up -d
  

• 验证集群状态（在任意VM执行）：

  # 创建测试主题，副本数设为2实现冗余
  docker run --rm apache/kafka:latest kafka-topics.sh --create --topic test-replica --partitions 3 --replication-factor 2 --bootstrap-server 192.168.1.10:9093
  # 查看主题分区分布
  docker run --rm apache/kafka:latest kafka-topics.sh --describe --topic test-replica --bootstrap-server 192.168.1.10:9093
  

兼容方案：ZooKeeper模式 #

若需兼容旧版本Kafka，可采用ZooKeeper集群搭配Kafka节点的架构：

1. VM1的docker-compose.yml配置 #

version: '3.8'
services:
  zookeeper1:
    image: zookeeper:3.8
    container_name: zookeeper1
    ports:
      - "2181:2181"
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=192.168.1.11:2888:3888;2181
    volumes:
      - ./zookeeper1-data:/data
      - ./zookeeper1-log:/datalog

  kafka1:
    image: apache/kafka:latest
    container_name: kafka1
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,192.168.1.11:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.10:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
    volumes:
      - ./kafka1-data:/var/lib/kafka/data
    depends_on:
      - zookeeper1

2. VM2的docker-compose.yml配置 #

version: '3.8'
services:
  zookeeper2:
    image: zookeeper:3.8
    container_name: zookeeper2
    ports:
      - "2181:2181"
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=192.168.1.10:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181
    volumes:
      - ./zookeeper2-data:/data
      - ./zookeeper2-log:/datalog

  kafka2:
    image: apache/kafka:latest
    container_name: kafka2
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: 192.168.1.10:2181,zookeeper2:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.11:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
    volumes:
      - ./kafka2-data:/var/lib/kafka/data
    depends_on:
      - zookeeper2

3. 启动与验证 #

执行docker-compose up -d启动集群，验证方式同KRaft模式。

扩展更多节点的方法 #

无论采用哪种模式，新增节点只需：

1. 分配唯一的NODE_ID（KRaft）或BROKER_ID（ZooKeeper）
2. 在配置中加入现有集群的控制器投票地址（KRaft）或ZooKeeper地址（ZooKeeper）
3. 设置advertised.listeners为新VM的IP
4. 启动新节点后，集群会自动识别并加入

关键注意事项 #

• 生产环境建议启用SSL加密监听，上述配置为简化演示使用PLAINTEXT
• 主题的replication-factor需设置为≥2，确保单节点故障时数据不丢失
• 所有节点必须使用相同的集群ID（KRaft）或ZooKeeper集群配置

内容的提问来源于stack exchange，提问作者Ram