You need to enable JavaScript to run this app.
优惠活动
大模型
产品
解决方案
定价
更多

Kafka KRaft集群异常：3个Broker始终有一个处于宕机状态

avatar
安全阿岳
2026-6-17
Kafka KRaft集群异常：Broker轮流宕机问题

问题现象

搭建3个Broker+3个Controller的KRaft集群时，出现Broker轮流宕机的情况：每次总有一个Broker处于宕机状态，恢复后另一个Broker又会宕机。

Docker配置 

version: "3.7"
services:
  controller-1:
    image: confluentinc/cp-kafka:latest
    hostname: controller-1
    container_name: controller-1
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093,3@controller-3:9093"
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER # Set your generated UUID her
    volumes:
      - controller_1_data:/var/lib/kafka/data
    user: 1000:1000
    networks:
      - kafka-connector
    restart: always

  controller-2:
    image: confluentinc/cp-kafka:latest
    hostname: controller-2
    container_name: controller-2
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_PROCESS_ROLES: controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093,3@controller-3:9093"
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
    volumes:
      - controller_2_data:/var/lib/kafka/data
    networks:
      - kafka-connector
    restart: always

  controller-3:
    image: confluentinc/cp-kafka:latest
    hostname: controller-3
    container_name: controller-3
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_PROCESS_ROLES: controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093,3@controller-3:9093"
      KAFKA_LISTENERS: CONTROLLER://0.0.0.0:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
    volumes:
      - controller_3_data:/var/lib/kafka/data
    networks:
      - kafka-connector
    restart: always

  kafka-1:
    image: confluentinc/cp-kafka:latest
    hostname: kafka-1
    container_name: kafka-1
    ports:
      - 29092:29092
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 11
      KAFKA_NODE_ID: 11
      KAFKA_PROCESS_ROLES: broker
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.18.2:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: "2@controller-2:9093,3@controller-3:9093"
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
    volumes:
      - kafka_1_data:/var/lib/kafka/data
    networks:
      - kafka-connector
    restart: always

  kafka-2:
    image: confluentinc/cp-kafka:latest
    hostname: kafka-2
    container_name: kafka-2
    ports:
      - 29093:29092
      - 9094:9092
    environment:
      KAFKA_BROKER_ID: 12
      KAFKA_NODE_ID: 12
      KAFKA_PROCESS_ROLES: broker
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.18.2:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,3@controller-3:9093"
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
    volumes:
      - kafka_2_data:/var/lib/kafka/data
    networks:
      - kafka-connector
    restart: always

  kafka-3:
    image: confluentinc/cp-kafka:latest
    hostname: kafka-3
    container_name: kafka-3
    ports:
      - 29094:29092
      - 9096:9092
    environment:
      KAFKA_BROKER_ID: 13
      KAFKA_NODE_ID: 13
      KAFKA_PROCESS_ROLES: broker
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.18.2:9096
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093"
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
    volumes:
      - kafka_3_data:/var/lib/kafka/data
    networks:
      - kafka-connector
    restart: always

  debezium:
    image: debezium/connect:2.4
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      - BOOTSTRAP_SERVERS=kafka-1:29092,kafka-2:29093,kafka-3:29094
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=my_connect_configs
      - OFFSET_STORAGE_TOPIC=my_connect_offsets
      - STATUS_STORAGE_TOPIC=my_connect_statuses
    ports:
      - 8083:8083
    networks:
      - kafka-connector
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "1g"
        max-file: "5"

  kafdrop:
    image: obsidiandynamics/kafdrop
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKER_CONNECT: kafka-1:9092,kafka-2:9094,kafka-3:9096
    networks:
      - kafka-connector
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "1g"
        max-file: "5"
volumes:
  controller_1_data: ~
  controller_2_data: ~
  controller_3_data: ~
  kafka_1_data: ~
  kafka_2_data: ~
  kafka_3_data: ~

networks:
  kafka-connector:
    driver: bridge

错误日志

Broker-3宕机时的核心错误 

[2024-10-18 11:39:13,063] ERROR Encountered fatal fault: Unexpected error in raft IO thread (org.apache.kafka.server.fault.ProcessTerminatingFaultHandler)
java.lang.IllegalStateException: Cannot transition to Follower with leaderId=3 and epoch=80 since it is not one of the voters [1, 2]
    at org.apache.kafka.raft.QuorumState.transitionToFollower(QuorumState.java:381)
    at org.apache.kafka.raft.KafkaRaftClient.transitionToFollower(KafkaRaftClient.java:518)
    at org.apache.kafka.raft.KafkaRaftClient.maybeTransition(KafkaRaftClient.java:1523)
    at org.apache.kafka.raft.KafkaRaftClient.maybeHandleCommonResponse(KafkaRaftClient.java:1473)
    at org.apache.kafka.raft.KafkaRaftClient.handleFetchResponse(KafkaRaftClient.java:1071)
    at org.apache.kafka.raft.KafkaRaftClient.handleResponse(KafkaRaftClient.java:1550)
    at org.apache.kafka.raft.KafkaRaftClient.handleInboundMessage(KafkaRaftClient.java:1676)
    at org.apache.kafka.raft.KafkaRaftClient.poll(KafkaRaftClient.java:2251)
    at kafka.raft.KafkaRaftManager$RaftIoThread.doWork(RaftManager.scala:64)
    at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:127)

Broker-2宕机时的核心错误 

[2024-10-18 11:43:54,404] ERROR Encountered fatal fault: Unexpected error in raft IO thread (org.apache.kafka.server.fault.ProcessTerminatingFaultHandler)
java.lang.IllegalStateException: Cannot transition to Follower with leaderId=2 and epoch=123 since it is not one of the voters [1, 3]

问题根因

每个Broker的KAFKA_CONTROLLER_QUORUM_VOTERS配置错误：

  • kafka-1只配置了controller-2和controller-3，缺少controller-1
  • kafka-2只配置了controller-1和controller-3，缺少controller-2
  • kafka-3只配置了controller-1和controller-2，缺少controller-3

这种配置导致Broker无法识别完整的Controller集群，当Controller集群选举出的leader不在Broker配置的voters列表中时，Broker就会抛出异常并终止进程。

修复方案

将所有Broker的KAFKA_CONTROLLER_QUORUM_VOTERS参数统一修改为和Controller一致的完整列表：

KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093,3@controller-3:9093"

修改后的Broker配置示例（以kafka-1为例） 

kafka-1:
  image: confluentinc/cp-kafka:latest
  hostname: kafka-1
  container_name: kafka-1
  ports:
    - 29092:29092
    - 9092:9092
  environment:
    KAFKA_BROKER_ID: 11
    KAFKA_NODE_ID: 11
    KAFKA_PROCESS_ROLES: broker
    KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.18.2:9092
    KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
    # 修正此处配置，包含所有3个Controller
    KAFKA_CONTROLLER_QUORUM_VOTERS: "1@controller-1:9093,2@controller-2:9093,3@controller-3:9093"
    KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
    KAFKA_LOG_DIRS: /var/lib/kafka/data
    CLUSTER_ID: KixvqJ76Qn-xLPPrSfBQSw
    KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
    KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
    KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
    KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
  volumes:
    - kafka_1_data:/var/lib/kafka/data
  networks:
    - kafka-connector
  restart: always

额外注意事项

  1. 修改配置后，需要清理所有Broker和Controller的持久化卷（旧的错误配置可能已写入本地存储），再重新启动集群：
    docker-compose down -v
docker-compose up -d
  2. 确保所有节点的CLUSTER_ID完全一致，这是KRaft集群的唯一标识，不一致会导致节点无法加入集群。

内容的提问来源于stack exchange，提问作者Furkan YIlmaZ

相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分)， 限时 9.9 元起

产品介绍相关文档
相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分)， 限时 9.9 元起

产品介绍相关文档
火山引擎 最新活动
全面开放
Doubao-Seedance-2.5全新模型升级

模型再升级,30秒超长叙事, 模态参考扩容

HOT
方舟Codingplan

模型自由，工具不限，最新支持 Deepseek-V4 系列、GLM-5.3 系列

HOT
方舟Agent plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列

HOT
新用户特惠专场

大模型19元起，Al应用9.9元畅享，新人首购爆款尽享优惠

最近更新时间：2026.06.17 05:19:49