Docker Compose部署Kafka集群重启失败：检查点文件损坏

问题现象 #

使用Docker Compose部署的Kafka集群首次启动正常，Windows机器重启后再次启动集群，Kafka容器启动失败，多次删除重建集群仍无法解决。

错误日志 #

2023-06-10 18:27:46 java.io.IOException: Malformed line in checkpoint file [/var/lib/kafka/data/log-start-offset-checkpoint]: 
2023-06-10 18:27:46     at org.apache.kafka.server.common.CheckpointFile$CheckpointReadBuffer.buildMalformedLineException(CheckpointFile.java:172)
2023-06-10 18:27:46     at org.apache.kafka.server.common.CheckpointFile$CheckpointReadBuffer.toInt(CheckpointFile.java:167)
2023-06-10 18:27:46     at org.apache.kafka.server.common.CheckpointFile$CheckpointReadBuffer.read(CheckpointFile.java:133)
2023-06-10 18:27:46     at org.apache.kafka.server.common.CheckpointFile.read(CheckpointFile.java:106)
2023-06-10 18:27:46     at kafka.server.checkpoints.CheckpointFileWithFailureHandler.read(CheckpointFileWithFailureHandler.scala:48)
2023-06-10 18:27:46     at kafka.server.checkpoints.OffsetCheckpointFile.read(OffsetCheckpointFile.scala:70)
2023-06-10 18:27:46     at kafka.log.LogManager.$anonfun$loadLogs$4(LogManager.scala:384)
2023-06-10 18:27:46     at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:563)
2023-06-10 18:27:46     at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:561)
2023-06-10 18:27:46     at scala.collection.AbstractIterable.foreach(Iterable.scala:926)
2023-06-10 18:27:46     at kafka.log.LogManager.loadLogs(LogManager.scala:353)
2023-06-10 18:27:46     at kafka.log.LogManager.startupWithConfigOverrides(LogManager.scala:522)
2023-06-10 18:27:46     at kafka.log.LogManager.startup(LogManager.scala:482)
2023-06-10 18:27:46     at kafka.server.KafkaServer.startup(KafkaServer.scala:263)
2023-06-10 18:27:46     at kafka.Kafka$.main(Kafka.scala:109)
2023-06-10 18:27:46     at kafka.Kafka.main(Kafka.scala)
2023-06-10 18:27:46 [2023-06-10 15:27:46,150] WARN Error occurred while reading log-start-offset-checkpoint file of directory /var/lib/kafka/data, resetting to the base offset of the first segment (kafka.log.LogManager)
2023-06-10 18:27:46 org.apache.kafka.common.errors.KafkaStorageException: Error while reading checkpoint file /var/lib/kafka/data/log-start-offset-checkpoint

Docker Compose配置 #

version: "2.1"
name: kafka-cluster-single
services:
  zoo1:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zoo1
    container_name: zoo1
    networks:
      - my-network
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
    volumes:
      - ~docker/volumes/kafka-cluster-single/zoo1/data:/var/lib/zookeeper/data
      - ~docker/volumes/kafka-cluster-single/zoo1/log:/var/lib/zookeeper/log

  zoo2:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zoo2
    container_name: zoo2
    networks:
      - my-network
    ports:
      - "2182:2182"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2182
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
    volumes:
      - ~docker/volumes/kafka-cluster-single/zoo2/data:/var/lib/zookeeper/data
      - ~docker/volumes/kafka-cluster-single/zoo2/log:/var/lib/zookeeper/log

  zoo3:
    image: confluentinc/cp-zookeeper:7.3.0
    hostname: zoo3
    container_name: zoo3
    networks:
      - my-network
    ports:
      - "2183:2183"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2183
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_SERVERS: zoo1:2888:3888;zoo2:2888:3888;zoo3:2888:3888
    volumes:
      - ~docker/volumes/kafka-cluster-single/zoo3/data:/var/lib/zookeeper/data
      - ~docker/volumes/kafka-cluster-single/zoo3/log:/var/lib/zookeeper/log

  kafka1:
    image: confluentinc/cp-kafka:7.3.0
    hostname: kafka1
    container_name: kafka1
    networks:
      - my-network
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
      KAFKA_CONFLUENT_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_NUM_PARTITIONS: "30"
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    volumes:
      - ~docker/volumes/kafka-cluster-single/kafka1/data:/var/lib/kafka/data

  connect:
    image: confluentinc/cp-kafka-connect:latest
    networks:
      - my-network
    depends_on:
      - zoo3
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "default"
      CONNECT_CONFIG_STORAGE_TOPIC: my_connect_configs
      CONNECT_OFFSET_STORAGE_TOPIC: my_connect_offsets
      CONNECT_STATUS_STORAGE_TOPIC: my_connect_statuses
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schemaregistry:8081"
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schemaregistry:8081"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "localhost"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: /usr/share/java,/etc/kafka-connect/jars
    volumes:
      - ~docker/volumes/kafka-cluster-single/kafka-connect/jars:/etc/kafka-connect/jars
      - ~docker/volumes/kafka-cluster-single/kafka-connect/etc/kafka:/etc/kafka

  kowl:
    image: quay.io/cloudhut/kowl:master
    depends_on:
      - kafka1
    networks:
      - my-network
    environment:
      KAFKA_BROKERS: kafka1:29092
      CONNECT_ENABLED: "true"
      CONNECT_CLUSTERS_NAME: "my-connect-cluster"
      CONNECT_CLUSTERS_URL: http://connect:8083/
      SCHEMAREGISTRY_ENABLED: "true"
    ports:
      - "8081:8080"
    volumes:
      - ~docker/volumes/kafka-cluster-single/kowl/etc/configs:/etc/kowl/configs
      - ~docker/volumes/kafka-cluster-single/kowl/etc/secrets:/etc/kowl/secrets
networks:
  my-network:
    external: true

解决方案 #

1. 修复损坏的检查点文件 #

日志明确指出log-start-offset-checkpoint文件格式损坏，这是由于Windows机器强制关机时Kafka未正常关闭，导致文件写入不完整。

• 进入Kafka数据挂载目录（Windows下路径为%USERPROFILE%\docker\volumes\kafka-cluster-single\kafka1\data）
• 删除log-start-offset-checkpoint文件，Kafka启动时会自动重建该文件

2. 避免强制关机导致的文件损坏 #

• 关机前先停止Docker Compose集群：执行docker-compose down命令，确保Kafka和ZooKeeper正常关闭
• 若使用Windows WSL2，需确保WSL2实例正常关闭后再关机

3. 优化Docker Compose配置（可选） #

• 为Kafka添加KAFKA_LOG_CLEANUP_POLICY: "delete"配置，避免日志文件过多积累
• 调整Kafka的关闭超时时间：添加KAFKA_SHUTDOWN_TIMEOUT_MS: "30000"，确保有足够时间完成文件写入

内容的提问来源于stack exchange，提问作者Zeevac