Spring Boot 2.7.5对接Azure Event Hub遇authBytes序列化空值异常求助

异常栈信息 #

java.lang.RuntimeException: non-nullable field authBytes was serialized as null
    at org.apache.kafka.common.message.SaslAuthenticateResponseData.read(SaslAuthenticateResponseData.java:137) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.message.SaslAuthenticateResponseData.<init>(SaslAuthenticateResponseData.java:86) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.requests.SaslAuthenticateResponse.parse(SaslAuthenticateResponse.java:76) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.requests.AbstractResponse.parseResponse(AbstractResponse.java:187) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.requests.AbstractResponse.parseResponse(AbstractResponse.java:109) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.NetworkClient.parseResponse(NetworkClient.java:729) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.security.authenticator.SaslClientAuthenticator.receiveKafkaResponse(SaslClientAuthenticator.java:576) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.security.authenticator.SaslClientAuthenticator.receiveToken(SaslClientAuthenticator.java:508) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.security.authenticator.SaslClientAuthenticator.authenticate(SaslClientAuthenticator.java:302) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:181) ~[kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:543) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.common.network.Selector.poll(Selector.java:481) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:560) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.producer.internals.Sender.awaitNodeReady(Sender.java:526) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.producer.internals.Sender.maybeSendAndPollTransactionalRequest(Sender.java:447) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:316) [kafka-clients-3.1.2.jar:na]
    at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:243) [kafka-clients-3.1.2.jar:na]
    at java.lang.Thread.run(Thread.java:748) [na:1.8.0_202]

当前配置 #

application.properties #

server:
  port: 8088
spring:
  kafka:
    client-id: first-producer
    properties:
      sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="$ConnectionString" password="SOME VALUE";
      sasl.mechanism: PLAIN
      security.protocol: SASL_SSL
    consumer:
      bootstrap-servers: AAA.servicebus.windows.net:9093
      group-id: group-id
      auto-offset-reset: earliest
      enable-auto-commit: true
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
    producer:
      bootstrap-servers: AAA.servicebus.windows.net:9093
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
topic:
  name: t-employee

pom.xml #

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.7.5</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.example</groupId>
    <artifactId>kafka-producer-consumer</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>kafka-producer-consumer</name>
    <description>Demo project for Spring Boot</description>
    
    <properties>
        <java.version>1.8</java.version>
    </properties>
    
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.datatype</groupId>
            <artifactId>jackson-datatype-jsr310</artifactId>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

解决方法 #

• 修正生产者序列化配置：生产者应使用序列化器（Serializer）而非反序列化器（Deserializer），修改application.properties中的生产者配置：

  producer:
    bootstrap-servers: AAA.servicebus.windows.net:9093
    key-serializer: org.apache.kafka.common.serialization.StringSerializer
    value-serializer: org.apache.kafka.common.serialization.StringSerializer
  

• 升级Kafka客户端版本：Spring Boot 2.7.5默认的kafka-clients 3.1.2与Azure Event Hub存在SASL兼容性问题，在pom.xml中指定更高兼容版本：

  <properties>
      <java.version>1.8</java.version>
      <kafka.version>3.2.3</kafka.version>
  </properties>
  

• 验证JAAS配置格式：确保sasl.jaas.config中的password为Azure Event Hub完整连接字符串，格式正确：

  sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="$ConnectionString" password="Endpoint=sb://AAA.servicebus.windows.net/;SharedAccessKeyName=XXX;SharedAccessKey=XXX";
  

• 禁用不必要的事务：若无需事务支持，在生产者配置中关闭事务前缀，避免触发额外SASL验证：

  producer:
    transaction-id-prefix: ""
  

内容的提问来源于stack exchange，提问作者Jeff Cook