如何使用Flutter抓取懒加载网站的全部图片与链接？

方法一：利用网站分页机制直接爬取多页 #

该网站URL带有?page=1参数，说明采用分页加载逻辑，直接循环请求不同页码的页面即可获取全部内容。

修改代码如下：

class Top2 extends StatefulWidget {
  const Top2({Key? key}) : super(key: key);

  @override
  State<Top2> createState() => _Top2State();
}

class _Top2State extends State<Top2> {
  List<Map<String, dynamic>> top2Wall = [];
  bool top2Loaded = false;
  int currentPage = 1;
  bool hasMore = true;

  Future<void> fetchPage(int page) async {
    final top2Scraper = WebScraper('https://mobile.alphacoders.com');
    if (await top2Scraper.loadWebPage('/by-category/3?page=$page')) {
      final pageData = top2Scraper.getElement(
          'div.container-masonry > div.item > a > img.img-responsive',
          ['src', 'title']);
      if (pageData.isEmpty) {
        hasMore = false;
        return;
      }
      setState(() {
        top2Wall.addAll(pageData);
      });
    } else {
      hasMore = false;
    }
  }

  void top2Fetch() async {
    // 循环请求页码，直到无更多内容
    while (hasMore) {
      await fetchPage(currentPage);
      currentPage++;
      // 添加延迟避免被网站封禁
      await Future.delayed(const Duration(seconds: 1));
    }
    setState(() {
      top2Loaded = true;
    });
  }

  @override
  void initState() {
    super.initState();
    top2Fetch();
  }

  @override
  Widget build(BuildContext context) {
    Size screenSize = MediaQuery.of(context).size;
    return Scaffold(
      body: top2Loaded
          ? Container(
              height: screenSize.height,
              width: double.infinity,
              child: SingleChildScrollView(
                physics: const BouncingScrollPhysics(),
                child: Wrap(
                  children: [
                    for (int i = 0; i < top2Wall.length; i++)
                      WallCard(src: top2Wall[i]['attributes']['src'])
                  ],
                ),
              ),
            )
          : const Center(
              child: CircularProgressIndicator(color: Colors.cyanAccent),
            ),
    );
  }
}

方法二：使用支持JS渲染的组件模拟滚动加载 #

如果网站无明显分页，仅靠滚动触发懒加载，可借助flutter_inappwebview加载页面并模拟滚动，触发内容加载后抓取DOM。

1. 添加依赖到pubspec.yaml：

dependencies:
  flutter_inappwebview: ^6.0.0

2. 实现代码：

import 'package:flutter_inappwebview/flutter_inappwebview.dart';

class Top2 extends StatefulWidget {
  const Top2({Key? key}) : super(key: key);

  @override
  State<Top2> createState() => _Top2State();
}

class _Top2State extends State<Top2> {
  List<Map<String, dynamic>> top2Wall = [];
  bool top2Loaded = false;
  late InAppWebViewController webViewController;

  void top2Fetch() async {
    await webViewController.loadUrl(
      urlRequest: URLRequest(url: Uri.parse('https://mobile.alphacoders.com/by-category/3')),
    );

    // 多次模拟滚动触发懒加载，次数可根据实际情况调整
    for (int i = 0; i < 5; i++) {
      await webViewController.evaluateJavascript(
        source: "window.scrollTo(0, document.body.scrollHeight);",
      );
      await Future.delayed(const Duration(seconds: 2)); // 等待内容加载完成
    }

    // 抓取所有图片数据
    final result = await webViewController.evaluateJavascript(
      source: """
        const imgs = document.querySelectorAll('div.container-masonry > div.item > a > img.img-responsive');
        return Array.from(imgs).map(img => ({src: img.src, title: img.title}));
      """,
    );

    setState(() {
      top2Wall = List<Map<String, dynamic>>.from(result);
      top2Loaded = true;
    });
  }

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      body: top2Loaded
          ? Container(
              height: MediaQuery.of(context).size.height,
              width: double.infinity,
              child: SingleChildScrollView(
                physics: const BouncingScrollPhysics(),
                child: Wrap(
                  children: [
                    for (var wall in top2Wall)
                      WallCard(src: wall['src'])
                  ],
                ),
              ),
            )
          : InAppWebView(
              initialUrlRequest: URLRequest(url: Uri.parse('https://mobile.alphacoders.com/by-category/3')),
              onWebViewCreated: (controller) {
                webViewController = controller;
                top2Fetch();
              },
            ),
    );
  }
}

方法三：直接抓取网站API接口（最推荐） #

打开浏览器开发者工具，查看网络请求中的XHR接口，找到网站加载图片的后端API，直接请求接口获取JSON数据，效率远高于DOM抓取。

示例代码（需根据实际接口调整参数）：

import 'dart:convert';
import 'package:http/http.dart' as http;

class Top2 extends StatefulWidget {
  const Top2({Key? key}) : super(key: key);

  @override
  State<Top2> createState() => _Top2State();
}

class _Top2State extends State<Top2> {
  List<Map<String, dynamic>> top2Wall = [];
  bool top2Loaded = false;
  int currentPage = 1;
  bool hasMore = true;

  Future<void> fetchApiPage(int page) async {
    // 实际接口需从浏览器开发者工具中获取
    final response = await http.get(
      Uri.parse('https://mobile.alphacoders.com/api/get_category_items?category_id=3&page=$page'),
    );

    if (response.statusCode == 200) {
      final data = jsonDecode(response.body);
      final items = data['items'] as List;
      if (items.isEmpty) {
        hasMore = false;
        return;
      }
      setState(() {
        top2Wall.addAll(items.map((item) => {
          'src': item['thumb_url'],
          'title': item['title']
        }).toList());
      });
    } else {
      hasMore = false;
    }
  }

  void top2Fetch() async {
    while (hasMore) {
      await fetchApiPage(currentPage);
      currentPage++;
      await Future.delayed(const Duration(seconds: 1));
    }
    setState(() {
      top2Loaded = true;
    });
  }

  @override
  void initState() {
    super.initState();
    top2Fetch();
  }

  @override
  Widget build(BuildContext context) {
    Size screenSize = MediaQuery.of(context).size;
    return Scaffold(
      body: top2Loaded
          ? Container(
              height: screenSize.height,
              width: double.infinity,
              child: SingleChildScrollView(
                physics: const BouncingScrollPhysics(),
                child: Wrap(
                  children: [
                    for (var wall in top2Wall)
                      WallCard(src: wall['src'])
                  ],
                ),
              ),
            )
          : const Center(
              child: CircularProgressIndicator(color: Colors.cyanAccent),
            ),
    );
  }
}

内容的提问来源于stack exchange，提问作者Cheems Monarch