同一URL下分页表格爬取问题求助（附代码尝试）

问题描述 #

需要爬取目标网站的分页表格，该网站所有页面URL保持不变。尝试过多个Selenium方案，但要么运行几页后超时，要么持续爬取同一页面；也了解到可提取JSON文件实现，但不知具体操作。

尝试过的代码 #

Code 1 #

from lxml import html
import requests
import pandas as pd
import re
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from openpyxl import load_workbook
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.action_chains import ActionChains
#from .actions.wheel_input import ScrollOrigin
from win32com.client import Dispatch
from geopy.geocoders import Nominatim
from geopy.extra.rate_limiter import RateLimiter
import urllib
import requests

options = Options()
# options.add_argument('--headless')
#options.add_argument("start-maximized")
options.add_argument('disable-infobars')
driver=webdriver.Chrome(options=options)

url = 'https://tariffs.ib-net.org/sites/IBNET/TariffTable#'
driver.get(url)
time.sleep(10)
wait = WebDriverWait(driver, 10)
x=driver.find_element(By.XPATH,'//*[@id="datatab_length"]/label/select')
drop=Select(x)

drop.select_by_visible_text("100")
time.sleep(10)

table = wait.until(EC.presence_of_element_located((By.XPATH, "//table[@class='table table-striped table-hover table-bordered dataTable no-footer dtr-inline collapsed']")))

#utility=[]
#city=[]
#service=[]
#date=[]
#fifteenm3=[]
#fiftym3 = []
#hundredm3 = []
data_list=[]
while True:
    # Extract data from the current page
    rows = table.find_elements(By.XPATH, "//table[@class='table table-striped table-hover table-bordered dataTable no-footer dtr-inline collapsed']//tbody")
    
    for row in rows:
        columns = row.find_elements(By.TAG_NAME, "tr")
        data_list.append([col.text.strip() for col in columns])
        print(data_list)
        next_button = driver.find_element(By.XPATH, "//*[@class='paginate_button next']/a")
        if next_button:
             # Click the next page button
            next_button.click()
            time.sleep(10)
            continue
        else:
            break 

Code 2 #

from lxml import html
import requests
import pandas as pd
import re
import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from openpyxl import load_workbook
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.action_chains import ActionChains
from win32com.client import Dispatch
from geopy.geocoders import Nominatim
from geopy.extra.rate_limiter import RateLimiter
import urllib
import requests


options = webdriver.ChromeOptions() 
options.add_argument('--start-maximized')
options.add_argument("disable-gpu")
browser = webdriver.Chrome()
browser.maximize_window()
actions=ActionChains(browser)
browser.get("https://tariffs.ib-net.org/sites/IBNET/TariffTable#")
time.sleep(5)

table_header= browser.find_elements(By.XPATH,"//table[@id='datatab']/thead")
header_row = []
for header in table_header:
    header_row.append(header.text)
#print(header_row)

utility=[]
city=[]
service=[]
date=[]
fifteenm3=[]
fiftym3=[]
hundredm3=[]

while True:
    all_rows = browser.find_elements(By.XPATH,"//div[@class='row']//tbody")
    for index in range(len(all_rows)):
        all_columns = all_rows[index].find_elements(By.XPATH,"//*[@role='row']")
        utility.append(all_columns[0].text)
        print(utility)
        city.append(all_columns[1].text)
        print(city)
        service.append(all_columns[2].text)
        print(service)
        date.append(all_columns[3].text)
        print(date)
        fifteenm3.append(all_columns[4].text)
        print(fifteenm3)
        fiftym3.append(all_columns[5].text)
        print(fiftym3)
        hundredm3.append(all_columns[6].text)
        print(hundredm3)
    if browser.find_element(By.XPATH,"//*[@class='paginate_button next']/a"):
        browser.find_element(By.XPATH,"//*[@class='paginate_button next']/a").click()
        time.sleep(5)
        continue
    else:
        break
        
df=pd.DataFrame()
df['Utlity']=utility
df['service']=service
df['city']=city
df['date']=date
df['15m3']=fifteenm3
df['50m3']=fiftym3
df['100m3']=hundredm3

df.to_csv('data.csv')

解决方案 #

方案一：修复Selenium代码 #

原代码存在元素定位错误、未等待页面加载完成、循环逻辑混乱等问题，修复后的代码如下：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
import pandas as pd
import time

# 初始化浏览器
options = webdriver.ChromeOptions()
options.add_argument('--start-maximized')
driver = webdriver.Chrome(options=options)
wait = WebDriverWait(driver, 15)

url = 'https://tariffs.ib-net.org/sites/IBNET/TariffTable#'
driver.get(url)

# 设置每页显示100条数据
wait.until(EC.presence_of_element_located((By.ID, "datatab_length")))
per_page_select = Select(driver.find_element(By.ID, "datatab_length").find_element(By.TAG_NAME, "select"))
per_page_select.select_by_visible_text("100")
time.sleep(3)

# 初始化数据列表
data = []
headers = []

# 获取表头
header_rows = wait.until(EC.presence_of_element_located((By.XPATH, "//table[@id='datatab']/thead/tr")))
headers = [col.text.strip() for col in header_rows.find_elements(By.TAG_NAME, "th")]

while True:
    # 等待当前页表格加载完成
    wait.until(EC.presence_of_element_located((By.XPATH, "//table[@id='datatab']/tbody")))
    table_body = driver.find_element(By.XPATH, "//table[@id='datatab']/tbody")
    rows = table_body.find_elements(By.TAG_NAME, "tr")
    
    # 提取当前页数据
    for row in rows:
        cols = row.find_elements(By.TAG_NAME, "td")
        row_data = [col.text.strip() for col in cols]
        data.append(row_data)
    
    # 处理下一页按钮
    try:
        next_button = wait.until(EC.element_to_be_clickable((By.XPATH, "//a[@class='paginate_button next']")))
        # 检查按钮是否禁用（最后一页时会有disabled类）
        if 'disabled' in next_button.get_attribute('class'):
            break
        next_button.click()
        time.sleep(3)
    except:
        # 找不到下一页按钮或无法点击，退出循环
        break

# 保存数据到CSV
df = pd.DataFrame(data, columns=headers)
df.to_csv('ibnet_tariffs.csv', index=False)

driver.quit()

修复要点：

• 用WebDriverWait替代固定time.sleep，确保元素加载完成后再操作
• 精准定位表格和行元素，避免重复抓取同一页面数据
• 检查下一页按钮的disabled属性，准确判断是否到达最后一页
• 移除不必要的导入模块，简化代码结构

方案二：直接抓取后端API数据（推荐） #

该网站使用DataTables插件，数据通过AJAX请求加载，直接抓取API无需模拟浏览器，速度更快且更稳定：

import requests
import pandas as pd

# API请求参数
url = "https://tariffs.ib-net.org/sites/IBNET/TariffTable/DataTableServerSide"
params = {
    "draw": 1,
    "columns[0][data]": "UtilityName",
    "columns[0][name]": "",
    "columns[0][searchable]": "true",
    "columns[0][orderable]": "true",
    "columns[0][search][value]": "",
    "columns[0][search][regex]": "false",
    "columns[1][data]": "CityName",
    "columns[1][name]": "",
    "columns[1][searchable]": "true",
    "columns[1][orderable]": "true",
    "columns[1][search][value]": "",
    "columns[1][search][regex]": "false",
    "columns[2][data]": "ServiceName",
    "columns[2][name]": "",
    "columns[2][searchable]": "true",
    "columns[2][orderable]": "true",
    "columns[2][search][value]": "",
    "columns[2][search][regex]": "false",
    "columns[3][data]": "EffectiveDate",
    "columns[3][name]": "",
    "columns[3][searchable]": "true",
    "columns[3][orderable]": "true",
    "columns[3][search][value]": "",
    "columns[3][search][regex]": "false",
    "columns[4][data]": "Consumption15",
    "columns[4][name]": "",
    "columns[4][searchable]": "true",
    "columns[4][orderable]": "true",
    "columns[4][search][value]": "",
    "columns[4][search][regex]": "false",
    "columns[5][data]": "Consumption50",
    "columns[5][name]": "",
    "columns[5][searchable]": "true",
    "columns[5][orderable]": "true",
    "columns[5][search][value]": "",
    "columns[5][search][regex]": "false",
    "columns[6][data]": "Consumption100",
    "columns[6][name]": "",
    "columns[6][searchable]": "true",
    "columns[6][orderable]": "true",
    "columns[6][search][value]": "",
    "columns[6][search][regex]": "false",
    "order[0][column]": "0",
    "order[0][dir]": "asc",
    "start": 0,
    "length": 100,  # 每页100条
    "search[value]": "",
    "search[regex]": "false"
}

all_data = []
total_records = None

while True:
    response = requests.get(url, params=params)
    json_data = response.json()
    
    if total_records is None:
        total_records = json_data['recordsTotal']
    
    all_data.extend(json_data['data'])
    
    # 更新start参数，获取下一页数据
    params['start'] += params['length']
    params['draw'] += 1
    
    if params['start'] >= total_records:
        break

# 转换为DataFrame并保存
df = pd.DataFrame(all_data)
# 重命名列名匹配页面显示
df.columns = ['Utility', 'City', 'Service', 'Effective Date', '15m³', '50m³', '100m³']
df.to_csv('ibnet_tariffs_api.csv', index=False)

操作说明：

• 通过浏览器开发者工具的Network面板，找到AJAX请求的URL和参数
• 循环请求所有分页数据，直到获取完所有记录
• 无需模拟浏览器，避免Selenium的超时、重复抓取等问题

内容的提问来源于stack exchange，提问作者r_sh_07