Flutter无法抓取tefas.gov.tr网站表格数据的技术求助

核心问题分析 #

• 你当前用http.get获取的是网站静态初始HTML，而目标表格数据是通过AJAX异步请求加载的，初始HTML里根本没有tr.odd这类表格行元素。
• 代码里的await Future.delayed完全无效：延迟是在请求完成后执行，此时已经拿到静态页面，不会再触发动态数据加载。

解决方案一：直接调用网站数据API（推荐） #

tefas.gov.tr的历史数据是通过后台API接口返回的，直接请求API比解析HTML高效且可靠：

1. 抓包获取API信息
   打开浏览器开发者工具（F12）→ Network标签，筛选XHR请求，找到加载表格数据的接口（通常是GetHistory这类POST请求），记录：

   • 请求URL：https://www.tefas.gov.tr/DB/TarihselVeriler.aspx/GetHistory
   • 请求头：需包含Content-Type: application/json; charset=utf-8、Referer: https://www.tefas.gov.tr/TarihselVeriler.aspx
   • 请求体：可参考以下JSON（根据需求调整日期或数据量）：

     {"draw":1,"columns":[{"data":"TARIH","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"FON_KODU","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"FON_ADI","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"FIYAT","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"TUTAR","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"ADET","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":"TOPLAM","name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}}],"order":[{"column":0,"dir":"desc"}],"start":0,"length":10,"search":{"value":"","regex":false},"date":"18.08.2023"}
     

2. Flutter代码实现
   修改fetchData方法，直接请求API并解析JSON：

   import 'dart:convert';
   import 'package:http/http.dart' as http;
   
   // ... 其他代码不变
   
   Future<void> fetchData() async {
     try {
       final url = Uri.parse('https://www.tefas.gov.tr/DB/TarihselVeriler.aspx/GetHistory');
       final headers = {
         'Content-Type': 'application/json; charset=utf-8',
         'Referer': 'https://www.tefas.gov.tr/TarihselVeriler.aspx',
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
       };
       final requestBody = jsonEncode({
         "draw": 1,
         "columns": [
           {"data": "TARIH", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "FON_KODU", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "FON_ADI", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "FIYAT", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "TUTAR", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "ADET", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}},
           {"data": "TOPLAM", "name": "", "searchable": true, "orderable": true, "search": {"value": "", "regex": false}}
         ],
         "order": [{"column": 0, "dir": "desc"}],
         "start": 0,
         "length": 10, // 每页数据量，可调整
         "search": {"value": "", "regex": false},
         "date": "18.08.2023" // 目标日期，可动态修改
       });
   
       final response = await http.post(url, headers: headers, body: requestBody);
   
       if (response.statusCode == 200) {
         final jsonData = jsonDecode(response.body);
         final dataList = jsonData['data'] as List;
   
         extractedData.clear();
         for (final item in dataList) {
           extractedData.add({
             'tarih': item['TARIH']?.trim() ?? '',
             'fonKodu': item['FON_KODU']?.trim() ?? '',
             'fonAdi': item['FON_ADI']?.trim() ?? '',
             'fiyat': item['FIYAT']?.trim() ?? '',
           });
         }
         setState(() {});
       } else {
         print('HTTP Error: ${response.statusCode}');
       }
     } catch (e) {
       print('Fetch Error: $e');
     }
   }
   

解决方案二：用WebView渲染后抓取DOM #

如果API接口有反爬限制，可以用webview_flutter库渲染页面，等数据加载完成后通过JS获取表格数据：

1. 添加依赖
   在pubspec.yaml中添加：

   dependencies:
     webview_flutter: ^4.4.0
   

2. 代码实现
   修改页面为WebView，监听页面加载完成后执行JS提取数据：

   import 'dart:convert';
   import 'package:webview_flutter/webview_flutter.dart';
   
   class TumKayitlarSayfasi extends StatefulWidget {
     @override
     _TumKayitlarSayfasiState createState() => _TumKayitlarSayfasiState();
   }
   
   class _TumKayitlarSayfasiState extends State<TumKayitlarSayfasi> {
     List<Map<String, String>> extractedData = [];
     late WebViewController _webViewController;
   
     @override
     Widget build(BuildContext context) {
       return Scaffold(
         appBar: AppBar(title: Text('Web Scraping Example')),
         body: Column(
           children: [
             Expanded(
               child: WebView(
                 initialUrl: 'https://www.tefas.gov.tr/TarihselVeriler.aspx',
                 javascriptMode: JavascriptMode.unrestricted,
                 onWebViewCreated: (controller) {
                   _webViewController = controller;
                 },
                 onPageFinished: (url) async {
                   // 等待几秒确保数据加载完成（可根据实际调整）
                   await Future.delayed(const Duration(seconds: 3));
                   // 执行JS获取表格行数据
                   final jsResult = await _webViewController.runJavascriptReturningResult('''
                     Array.from(document.querySelectorAll('tr.odd, tr.even')).map(row => {
                       const cols = row.querySelectorAll('td');
                       return {
                         tarih: cols[0].textContent.trim(),
                         fonKodu: cols[1].textContent.trim(),
                         fonAdi: cols[2].textContent.trim(),
                         fiyat: cols[3].textContent.trim()
                       };
                     }).filter(item => item.tarih !== '');
                   ''');
                   // 解析JS返回的JSON字符串
                   final List<dynamic> dataList = jsonDecode(jsResult);
                   extractedData = dataList.map((item) => Map<String, String>.from(item)).toList();
                   setState(() {});
                 },
               ),
             ),
             if (extractedData.isNotEmpty)
               Expanded(
                 child: ListView.builder(
                   itemCount: extractedData.length,
                   itemBuilder: (context, index) {
                     final data = extractedData[index];
                     return ListTile(
                       title: Text(data['tarih']!),
                       subtitle: Text('${data['fonKodu']} - ${data['fonAdi']}'),
                       trailing: Text(data['fiyat']!),
                     );
                   },
                 ),
               ),
           ],
         ),
       );
     }
   }
   

注意事项 #

• 网站周末不发布数据，测试时请选择工作日日期。
• 不要频繁请求API，避免触发网站反爬机制导致IP被封禁。
• 若API参数有变化，需重新抓包更新请求体。

内容的提问来源于stack exchange，提问作者eVAL onger