Firebase RTDB分页读取大数据过慢问题求助

问题背景 #

需要从Firebase RTDB的stations节点获取10MB大数据：

• 单次全量读取耗时10-12秒，但偶尔会触发Android应用OutOfMemory错误崩溃
• 采用分页读取方案（每次读取2000条，约2MB数据）后，单页请求仍耗时约10秒，总耗时从10秒增至50秒，分页反而大幅增加总耗时

数据结构 #

安全规则 #

当前分页逻辑代码 #

private fun fetchDayStationsPaged(dayTag: String, lastNodeId: String? = null, stations: MutableList<StationCloud> = mutableListOf(), callback: (data: List<StationCloud>, errorMessage: LoadError?) -> Unit){
    val path = String.format(TimelineManager.KEY_TIMELINE_STATIONS, dayTag)

    val query = if (lastNodeId == null)
        database
            .getReference(path)
            .orderByKey()
            .limitToFirst(2000) //1 node takes approx 1KB
    else
        database
            .getReference(path)
            .orderByKey()
            .startAfter(lastNodeId)
            .limitToFirst(2000)

    Timber.d("loader recursion stations $dayTag/${stations.size}")

    fetchDayStations(query) { data, errorMessage ->
        if (errorMessage == LoadError.NotExist)
            callback(stations, null)
        else if (errorMessage != null)
            callback(emptyList(), errorMessage)
        else {
            stations.addAll(data)
            fetchDayStationsPaged(dayTag, data.last().nodeId, stations, callback)
        }
    }
}

private var loaderDisposable: Disposable? = null

private fun fetchDayStations(ref: Query, callback: (data: List<StationCloud>, errorMessage: LoadError?) -> Unit){

    loaderDisposable?.dispose()
    loaderDisposable = FirebaseHelper
        .dbReadAsSingle(ref)
        .subscribeOn(AndroidSchedulers.mainThread())
        .observeOn(AndroidSchedulers.mainThread())
        .doOnDispose {
            callback(emptyList(), LoadError.Cancelled)
        }
        .subscribe ({ snapshot ->
            if (snapshot.exists()) {
                val stations = mutableListOf<StationCloud>()
                snapshot.children.forEach { item ->
                    item.getValue(StationCloud::class.java)?.also { station ->
                        stations.add(station.copy(nodeId = item.key))
                    }
                }
                Timber.d("loader fetchDayStations stations size = ${stations.size}")
                callback(stations.toList(), null)
            } else
                callback(emptyList(), LoadError.NotExist)
        }, {
            Timber.e(it)
            callback(emptyList(), LoadError.CantGet)
        })
}

private fun dbReadAsSingle(ref: Query): Single<DataSnapshot> {
    return Single.create { emitter ->
        ref.get().addOnCompleteListener { task ->
            Timber.d("runQueryCloudFirst task succeed = ${task.isSuccessful}")
            if (task.isSuccessful && emitter.isDisposed.not()){
                Timber.d("runQueryCloudFirst children size = ${task.result.childrenCount}")
                emitter.onSuccess(task.result)
            } else
                task.exception?.also {
                    //todo check a bug: timeout exception doesn't work when offline
                    //https://github.com/firebase/firebase-android-sdk/issues/5771
                    Timber.e(it, "runQueryCloudFirst")
                    emitter.onError(it)
                }
        }
    }
}

1. 并行发起多页请求，替代串行递归 #

当前代码为串行分页：必须等前一页请求完成才发起下一页，总耗时是单页耗时×页数。改成并行请求可大幅压缩总耗时：

• 先获取总节点数，计算需要请求的页数
• 同时发起多页请求（控制并发数在3-5，避免触发Firebase限流）
• 所有请求完成后合并数据，或边接收边更新UI

示例调整思路：

private fun fetchAllPagesParallel(dayTag: String, pageSize: Int = 2000) {
    val path = String.format(TimelineManager.KEY_TIMELINE_STATIONS, dayTag)
    val baseRef = database.getReference(path)
    
    // 先获取总节点数计算页数
    baseRef.get().addOnSuccessListener { totalSnapshot ->
        val totalCount = totalSnapshot.childrenCount
        val pageCount = (totalCount / pageSize).toInt() + 1
        
        val requests = mutableListOf<Single<List<StationCloud>>>()
        for (i in 0 until pageCount) {
            val startAtKey = if (i == 0) null else getNthKey(totalSnapshot, i * pageSize)
            val query = startAtKey?.let {
                baseRef.orderByKey().startAfter(it).limitToFirst(pageSize)
            } ?: baseRef.orderByKey().limitToFirst(pageSize)
            requests.add(fetchPageAsSingle(query))
        }
        
        // 并行执行所有请求
        Single.zip(requests) { results ->
            results.flatMap { it as List<StationCloud> }
        }.subscribeOn(Schedulers.io())
          .observeOn(AndroidSchedulers.mainThread())
          .subscribe { allStations ->
              // 处理完整数据
          }
    }
}

// 辅助方法：获取第N个节点的key
private fun getNthKey(snapshot: DataSnapshot, index: Long): String? {
    var count = 0L
    snapshot.children.forEach {
        if (count == index) return it.key
        count++
    }
    return null
}

// 封装单页请求为Single
private fun fetchPageAsSingle(query: Query): Single<List<StationCloud>> {
    return Single.create { emitter ->
        query.get().addOnSuccessListener { snapshot ->
            val stations = mutableListOf<StationCloud>()
            snapshot.children.forEach { item ->
                item.getValue(StationCloud::class.java)?.let {
                    stations.add(it.copy(nodeId = item.key))
                }
            }
            emitter.onSuccess(stations)
        }.addOnFailureListener {
            emitter.onError(it)
        }
    }
}

2. 调整线程调度，避免主线程阻塞 #

当前代码把网络请求放在主线程执行，会增加请求耗时并阻塞UI，需将网络请求移到IO线程：

// 修改fetchDayStations中的线程调度
loaderDisposable = FirebaseHelper
    .dbReadAsSingle(ref)
    .subscribeOn(Schedulers.io()) // 网络请求放IO线程
    .observeOn(AndroidSchedulers.mainThread()) // 回调回主线程更新UI
    // 其他逻辑不变

3. 优化数据解析效率 #

• 避免在主线程执行getValue和copy操作，把解析逻辑移到IO线程
• 用Gson批量解析替代遍历子节点，提升效率：

// 批量解析示例
val type = object : TypeToken<Map<String, StationCloud>>() {}.type
val stationMap = snapshot.getValue(type) as Map<String, StationCloud>
val stations = stationMap.map { (key, station) -> station.copy(nodeId = key) }

4. 检查索引与安全规则 #

• 确认stations节点的orderByKey索引有效：Firebase默认对key建立索引，但复杂安全规则可能导致索引失效，服务器需全量扫描后分页，导致单页耗时与全量一致
• 简化安全规则：避免遍历节点做权限校验，尽量使用路径/属性直接校验，减少服务器处理时间

5. 启用本地持久化缓存 #

开启持久化后，重复请求优先读取缓存，大幅减少网络耗时：

// 初始化Firebase时开启全局持久化
FirebaseDatabase.getInstance().setPersistenceEnabled(true)
// 对stations节点启用缓存同步
database.getReference(path).keepSynced(true)

6. 调整分页大小，减少请求次数 #

单页请求耗时与全量接近时，说明大部分耗时是连接建立、服务器初始化查询的开销。可适当增大分页大小（如从2000条增至5000条，对应5MB），减少总请求次数，但需注意不要触发OOM。

内容的提问来源于stack exchange，提问作者Konstantin Konopko