Flutter应用中Cloud Firestore获取10万条数据耗时过长的优化问询

核心问题分析 #

当前方案的瓶颈在于storeProductsSnapshot的全量查询，以及后续嵌套循环中频繁遍历产品列表匹配的低效操作。结合你的数据规模（10万条products、当前90家stores），可以从查询优化、数据结构重构、算法效率三个维度入手优化。

具体优化措施 #

1. 优化Firestore查询，减少数据传输耗时 #

（1）添加复合索引

确保Firestore中products集合存在is_exist + storeRef的复合索引，无索引的whereIn查询会触发全表扫描，大幅拖慢速度。

（2）字段投影，只获取需要的数据

不需要下载产品文档的所有字段，只查询代码中用到的product_id、is_exist、name、pImage、storeRef、price，减少数据传输体积：

final QuerySnapshot storeProductsSnapshot = await fireStore
    .collection('products')
    .where('is_exist', isEqualTo: true)
    .where('storeRef', whereIn: storeRefs)
    .select(['product_id', 'is_exist', 'name', 'pImage', 'storeRef', 'price']) // 仅获取必需字段
    .get();

（3）分批次获取（可选）

如果后续stores数量增加导致单次查询数据量过大，可以用分页查询分批次加载，避免一次性占用过多内存：

List<QueryDocumentSnapshot> allStoreProductDocs = [];
QuerySnapshot? snapshot;
do {
  Query query = fireStore
      .collection('products')
      .where('is_exist', isEqualTo: true)
      .where('storeRef', whereIn: storeRefs)
      .select(['product_id', 'is_exist', 'name', 'pImage', 'storeRef', 'price'])
      .limit(500); // 每次加载500条
  if (snapshot != null) {
    query = query.startAfterDocument(snapshot.docs.last);
  }
  snapshot = await query.get();
  allStoreProductDocs.addAll(snapshot.docs);
} while (snapshot.docs.isNotEmpty);

2. 重构数据结构，提升匹配效率 #

将storeProducts转换成按storeRef.path分组的Map，后续查找指定商店的产品时无需遍历全局列表：

// 替换原storeProducts列表生成逻辑
final Map<String, List<Product>> storeToProducts = {};
for (final productSnapshot in allStoreProductDocs) {
  final ref = productSnapshot.reference;
  final product = Product(
    productId: productSnapshot['product_id'].toString(),
    productRef: ref.id.toString(),
    isExist: productSnapshot['is_exist'] as bool,
    name: productSnapshot['name'].toString(),
    pImage: productSnapshot['pImage'].toString(),
    storeRef: productSnapshot['storeRef'].path.toString(),
    price: productSnapshot['price'].toString(),
  );
  // 按商店路径分组存储
  if (!storeToProducts.containsKey(product.storeRef)) {
    storeToProducts[product.storeRef] = [];
  }
  storeToProducts[product.storeRef]!.add(product);
}

同时修改_getSimilarProductFromList的调用逻辑，直接传入当前商店的产品列表，减少无效遍历：

final Product? productInStore = _getSimilarProductFromList(
  product: product,
  storeProducts: storeToProducts[storeRef.path] ?? [], // 仅传入当前商店的产品
);

3. 用户产品查询优化 #

对用户产品查询同样使用字段投影，避免冗余数据传输：

// 用户产品列表查询添加投影
QuerySnapshot productListSnapshot = await fireStore
    .collection('mylist')
    .doc(listId)
    .collection('user_product_list')
    .select(['quantity', 'product_ref']) // 只取需要的字段
    .get();

// 批量获取用户产品时也添加字段投影
final List<Product> userProducts = await Future.wait(
  productRefs.map(
    (ref) async {
      final productSnapshot = await fireStore.collection('products').doc(ref.id)
          .select(['product_id', 'is_exist', 'name', 'pImage', 'storeRef', 'price'])
          .get();
      return Product(
        productId: productSnapshot['product_id'].toString(),
        productRef: ref.id.toString(),
        isExist: productSnapshot['is_exist'] as bool,
        name: productSnapshot['name'].toString(),
        pImage: productSnapshot['pImage'].toString(),
        storeRef: productSnapshot['storeRef'].path.toString(),
        price: productSnapshot['price'].toString(),
      );
    },
  ).toList(),
);

4. 优化嵌套循环逻辑 #

将用户产品与对应数量关联成Map，避免每次循环通过索引查找：

// 关联用户产品与数量
final Map<String, (Product, int)> userProductWithQuantity = {};
for (var i = 0; i < productListSnapshot.size; i++) {
  final productSnapshot = productListSnapshot.docs[i];
  final product = userProducts[i];
  userProductWithQuantity[product.productId] = (product, productSnapshot['quantity']);
}

// 简化嵌套循环逻辑
for (var storeRef in storeRefs) {
  final storePath = storeRef.path;
  final currentStoreProducts = storeToProducts[storePath] ?? [];
  double total = 0;
  for (var entry in userProductWithQuantity.entries) {
    final (product, quantity) = entry.value;
    final productId = entry.key;

    if (product.isExist && product.storeRef == storePath) {
      total += quantity * double.parse(product.minPrice);
      continue;
    }

    final Product? productInStore = _getSimilarProductFromList(
      product: product,
      storeProducts: currentStoreProducts,
    );
    if (productInStore != null) {
      total += quantity * double.parse(productInStore.minPrice);
      // 处理匹配后的轻量操作
    } else {
      // 处理无匹配的轻量操作
    }
  }
  storeTotal[storeRef] = total;
}

完整优化后代码示例 #

Future<void> rankStoresByPriceTotal2({double radius = 6}) async {
  final listId = await prefs.then((prefs) => prefs.getUserListId());

  try {
    final List<Stores> nearbyStores = await getNearbyStores(
      radius: radius,
      firestore: fireStore,
      userLocation: const GeoPoint(20.68016662, -103.3822084),
    );

    final List<DocumentReference> storeRefs = nearbyStores
        .map((store) => fireStore.collection(StoreConstant.storeCollection).doc(store.id))
        .toList();

    // 分批次+字段投影获取商店产品
    List<QueryDocumentSnapshot> allStoreProductDocs = [];
    QuerySnapshot? snapshot;
    do {
      Query query = fireStore
          .collection('products')
          .where('is_exist', isEqualTo: true)
          .where('storeRef', whereIn: storeRefs)
          .select(['product_id', 'is_exist', 'name', 'pImage', 'storeRef', 'price'])
          .limit(500);
      if (snapshot != null) {
        query = query.startAfterDocument(snapshot.docs.last);
      }
      snapshot = await query.get();
      allStoreProductDocs.addAll(snapshot.docs);
    } while (snapshot.docs.isNotEmpty);

    // 按storeRef分组存储产品
    final Map<String, List<Product>> storeToProducts = {};
    for (final productSnapshot in allStoreProductDocs) {
      final ref = productSnapshot.reference;
      final product = Product(
        productId: productSnapshot['product_id'].toString(),
        productRef: ref.id.toString(),
        isExist: productSnapshot['is_exist'] as bool,
        name: productSnapshot['name'].toString(),
        pImage: productSnapshot['pImage'].toString(),
        storeRef: productSnapshot['storeRef'].path.toString(),
        price: productSnapshot['price'].toString(),
      );
      if (!storeToProducts.containsKey(product.storeRef)) {
        storeToProducts[product.storeRef] = [];
      }
      storeToProducts[product.storeRef]!.add(product);
    }

    // 用户产品列表查询添加字段投影
    QuerySnapshot productListSnapshot = await fireStore
        .collection('mylist')
        .doc(listId)
        .collection('user_product_list')
        .select(['quantity', 'product_ref'])
        .get();

    final List<DocumentReference> productRefs = productListSnapshot.docs
        .map((productSnapshot) => productSnapshot['product_ref'] as DocumentReference)
        .toList();

    // 批量获取用户产品时使用字段投影
    final List<Product> userProducts = await Future.wait(
      productRefs.map(
        (ref) async {
          final productSnapshot = await fireStore.collection('products').doc(ref.id)
              .select(['product_id', 'is_exist', 'name', 'pImage', 'storeRef', 'price'])
              .get();
          return Product(
            productId: productSnapshot['product_id'].toString(),
            productRef: ref.id.toString(),
            isExist: productSnapshot['is_exist'] as bool,
            name: productSnapshot['name'].toString(),
            pImage: productSnapshot['pImage'].toString(),
            storeRef: productSnapshot['storeRef'].path.toString(),
            price: productSnapshot['price'].toString(),
          );
        },
      ).toList(),
    );

    // 关联用户产品与数量
    final Map<String, (Product, int)> userProductWithQuantity = {};
    for (var i = 0; i < productListSnapshot.size; i++) {
      final productSnapshot = productListSnapshot.docs[i];
      final product = userProducts[i];
      userProductWithQuantity[product.productId] = (product, productSnapshot['quantity']);
    }

    final Map<DocumentReference, double> storeTotal = {};

    // 简化嵌套循环计算总价
    for (var storeRef in storeRefs) {
      final storePath = storeRef.path;
      final currentStoreProducts = storeToProducts[storePath] ?? [];
      double total = 0;
      for (var entry in userProductWithQuantity.entries) {
        final (product, quantity) = entry.value;
        final productId = entry.key;

        if (product.isExist && product.storeRef == storePath) {
          total += quantity * double.parse(product.minPrice);
          continue;
        }

        final Product? productInStore = _getSimilarProductFromList(
          product: product,
          storeProducts: currentStoreProducts,
        );
        if (productInStore == null) {
          // 处理无匹配的轻量操作
        } else {
          total += quantity * double.parse(productInStore.minPrice);
          // 处理匹配后的轻量操作
        }
      }
      storeTotal[storeRef] = total;
    }

    // 排序逻辑...

  } catch (e) {
    print('Error ranking stores: $e');
  }
}

内容的提问来源于stack exchange，提问作者Rahul Gohil