已加锁对象抛出ConcurrentModificationException问题排查

应用运行一晚后，次日发现如下异常栈：

java.util.ConcurrentModificationException: null
at java.base/java.util.LinkedHashMap$LinkedHashIterator.nextNode(LinkedHashMap.java:756) ~[na:na]
    at java.base/java.util.LinkedHashMap$LinkedEntryIterator.next(LinkedHashMap.java:788) ~[na:na]
    at java.base/java.util.LinkedHashMap$LinkedEntryIterator.next(LinkedHashMap.java:786) ~[na:na]
    at ****.MyClass.cleanup(MyClass.kt:150) ~[main/:na]
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:na]
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[na:na]
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:na]
    at java.base/java.lang.reflect.Method.invoke(Method.java:568) ~[na:na]
    at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) ~[spring-context-6.0.12.jar:6.0.12]
    at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) ~[spring-context-6.0.12.jar:6.0.12]
    at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:96) ~[spring-context-6.0.12.jar:6.0.12]
    at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[na:na]
    at java.base/java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:264) ~[na:na]
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java) ~[na:na]
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[na:na]
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[na:na]
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[na:na]
    at java.base/java.lang.Thread.run(Thread.java:833) ~[na:na]

简化后的核心代码如下：

@Configuration
@EnableConfigurationProperties(MyConnectionProperties::class)
class MyClass(
    private val myProperties: MyConnectionProperties,
) {

    private val logger: KLogger = KotlinLogging.logger {}
    private val myLock = ReentrantLock()
    private val myHolder: MutableMap<String, MyConnectionExtension> = mutableMapOf()
     
    @Scheduled(cron = "...")
    fun cleanup() {
        val connectionsToClose: MutableList<LDAPConnection> = mutableListOf()

        myLock.withLock {
            myHolder.forEach { (userDn, connectionExtension) ->
                val lastTouchTime = max(
                    connectionExtension.connection.lastCommunicationTime,
                    connectionExtension.lastPullingTime,
                )
                val connectionName = connectionExtension.connection.connectionName

                if (lastTouchTime + myProperties.lifetimeMs < Instant.now().toEpochMilli()) {
                    myHolder.remove(userDn)
                    connectionsToClose.add(connectionExtension.connection)
                }
            }
        }

        connectionsToClose.forEach { connection ->
            try {
                connection.close()               
            } catch (e: Exception) {
                logger.warn(e) {"....."}
            }
        }

        logger.trace { "job finished" }
    }
    
    private fun getConnection(userId: String, password: String): LDAPConnection {
        myLock.withLock {
            val connectionExtension = myHolder[userId]
            val connection = connectionExtension?.connection

            return if (connection == null || !connection.isConnected) {
                createConnection(userId, password).also {
                    myHolder[userId] = MyConnectionExtension(
                        connection = it,
                        lastPullingTime = Instant.now().toEpochMilli(),
                    )
                }
            } else {                
                connectionExtension.lastPullingTime = Instant.now().toEpochMilli()
                connection
            }
        }
    }
    ...
}

已用ReentrantLock保护对myHolder的所有操作，仅cleanup和getConnection两个方法会修改或访问它，为何仍会出现并发修改异常？

根因分析 #

问题和多线程无关，是单线程下的遍历-修改冲突：

• myHolder默认是LinkedHashMap（Kotlin的mutableMapOf()默认实现），它的迭代器采用**快速失败（fail-fast）**机制。
• 在cleanup方法的forEach遍历过程中，直接调用myHolder.remove(userDn)修改了Map的结构，迭代器检测到结构变化后，就会抛出ConcurrentModificationException。
• 锁只能保证多线程安全，但管不了单线程内遍历过程中修改集合结构的问题。

解决方法 #

方案1：使用迭代器的remove方法（推荐） #

Kotlin的forEach底层用迭代器，但无法直接调用迭代器的remove，需显式获取迭代器操作：

myLock.withLock {
    val iterator = myHolder.entries.iterator()
    while (iterator.hasNext()) {
        val entry = iterator.next()
        val userDn = entry.key
        val connectionExtension = entry.value
        
        val lastTouchTime = max(
            connectionExtension.connection.lastCommunicationTime,
            connectionExtension.lastPullingTime,
        )

        if (lastTouchTime + myProperties.lifetimeMs < Instant.now().toEpochMilli()) {
            iterator.remove() // 用迭代器自身的remove方法，不会触发异常
            connectionsToClose.add(connectionExtension.connection)
        }
    }
}

方案2：先收集待删除的Key，再批量删除 #

遍历仅记录要删除的Key，遍历结束后统一删除：

myLock.withLock {
    val keysToRemove = mutableListOf<String>()
    myHolder.forEach { (userDn, connectionExtension) ->
        val lastTouchTime = max(
            connectionExtension.connection.lastCommunicationTime,
            connectionExtension.lastPullingTime,
        )

        if (lastTouchTime + myProperties.lifetimeMs < Instant.now().toEpochMilli()) {
            keysToRemove.add(userDn)
            connectionsToClose.add(connectionExtension.connection)
        }
    }
    keysToRemove.forEach { myHolder.remove(it) }
}

方案3：遍历Map的Key集合副本 #

通过复制Key集合，避免遍历原Map时修改结构：

myLock.withLock {
    // 复制keySet，遍历副本
    val userDns = myHolder.keys.toList()
    userDns.forEach { userDn ->
        val connectionExtension = myHolder[userDn] ?: return@forEach
        val lastTouchTime = max(
            connectionExtension.connection.lastCommunicationTime,
            connectionExtension.lastPullingTime,
        )

        if (lastTouchTime + myProperties.lifetimeMs < Instant.now().toEpochMilli()) {
            myHolder.remove(userDn)
            connectionsToClose.add(connectionExtension.connection)
        }
    }
}

内容的提问来源于stack exchange，提问作者gstackoverflow