Kafka-多个消费者

在 Kafka 中使用多个消费者对于处理大量数据非常有帮助。但是，如果不小心处理，可能会导致出现数据重复或丢失问题。以下是如何在 Java 中解决此问题的示例代码。

1. 创建 Topic：

bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic test

2. 生产者生产消息：

import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

public class SampleProducer {

    public static void main(String[] args) throws Exception {

        String topicName = "test";

        Properties props = new Properties();
        props.put("bootstrap.servers", "localhost:9092");
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        KafkaProducer<String, String> producer = new KafkaProducer<>(props);

        for (int i = 0; i < 100; i++) {
            producer.send(new ProducerRecord<>(topicName, Integer.toString(i), Integer.toString(i)));
        }

        producer.close();
    }

}

3. 消费者消费消息：

import java.util.Properties;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import java.util.ArrayList;
import java.util.List;

public class SampleConsumer {

    public static void main(String[] args) {

        String topicName = "test";
        int numConsumers = 3;

        Properties props = new Properties();
        props.put("bootstrap.servers", "localhost:9092");
        props.put("group.id", "testGroup");

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);

        List<TopicPartition> partitions = new ArrayList<>();
        int numPartitions = consumer.partitionsFor(topicName).size();
        for (int i = 0; i < numConsumers; i++) {
            partitions.add(new TopicPartition(topicName, i % numPartitions));
        }
        consumer.assign(partitions