如何优化Oddsportal赛事爬虫代码实现分页循环迭代遍历

• 自动抓取目标联赛下所有历史赛季的结果页链接，无需手动拼接赛季后缀
• 针对每个赛季结果页自动读取最大分页数量，自动生成全量分页URL，无需手动罗列页面编号
• 替换pandas已废弃的append方法为pd.concat，兼容更高版本pandas，同时优化数据合并效率
• 新增浏览器资源释放逻辑，避免后台残留Chrome进程

import pandas as pd
import time
from selenium import webdriver
from bs4 import BeautifulSoup as bs

# 基础配置
base_domain = "https://www.oddsportal.com"
# 要抓取的联赛首页地址，可替换为其他联赛
base_league_url = "https://www.oddsportal.com/soccer/england/premier-league/"
browser = webdriver.Chrome()
# 加隐式等待，避免页面没加载完成导致元素找不到
browser.implicitly_wait(10)


class GameData:
    def __init__(self):
        self.date = []
        self.time = []
        self.game = []
        self.score = []
        self.home_odds = []
        self.draw_odds = []
        self.away_odds = []
        self.country = []
        self.league = []


def parse_data(url):
    browser.get(url)
    # 加短延迟避免反爬
    time.sleep(1.5)
    df = pd.read_html(browser.page_source, header=0)[0]
    html = browser.page_source
    soup = bs(html, "lxml")
    cont = soup.find('div', {'id': 'wrap'})
    content = cont.find('div', {'id': 'col-content'})
    content = content.find('table', {'class': 'table-main'}, {'id': 'tournamentTable'})
    main = content.find('th', {'class': 'first2 tl'})
    if main is None:
        return None
    count = main.findAll('a')
    country = count[1].text
    league = count[2].text
    game_data = GameData()
    game_date = None
    for row in df.itertuples():
        if not isinstance(row[1], str):
            continue
        elif ':' not in row[1]:
            game_date = row[1].split('-')[0]
            continue
        game_data.date.append(game_date)
        game_data.time.append(row[1])
        game_data.game.append(row[2])
        game_data.score.append(row[3])
        game_data.home_odds.append(row[4])
        game_data.draw_odds.append(row[5])
        game_data.away_odds.append(row[6])
        game_data.country.append(country)
        game_data.league.append(league)
    return game_data


# 新增：获取指定联赛下所有历史赛季的结果页基础URL
def get_season_urls():
    browser.get(base_league_url)
    time.sleep(1)
    soup = bs(browser.page_source, "lxml")
    # 赛季选择下拉框的定位
    season_select = soup.find('div', class_='main-menu2 main-menu-gray')
    season_links = season_select.find_all('a')
    season_result_urls = []
    for link in season_links:
        season_url = base_domain + link['href'] + 'results/'
        season_result_urls.append(season_url)
    return season_result_urls


# 新增：获取指定赛季结果页的最大分页数量
def get_max_page(season_url):
    browser.get(season_url)
    time.sleep(1)
    soup = bs(browser.page_source, "lxml")
    pagination = soup.find('div', id='pagination')
    if not pagination:
        return 1
    page_nums = []
    for a in pagination.find_all('a'):
        text = a.text.strip()
        if text.isdigit():
            page_nums.append(int(text))
    return max(page_nums) if page_nums else 1


if __name__ == '__main__':
    # 存储所有页面的DataFrame
    df_list = []
    # 第一步：获取所有赛季的结果页基础地址
    season_urls = get_season_urls()
    for season_url in season_urls:
        # 第二步：获取当前赛季的最大分页
        max_page = get_max_page(season_url)
        # 遍历所有分页
        for page in range(1, max_page + 1):
            page_url = f"{season_url}#/page/{page}"
            print(f"正在抓取：{page_url}")
            game_data = parse_data(page_url)
            if game_data is None:
                continue
            page_df = pd.DataFrame(game_data.__dict__)
            df_list.append(page_df)
    
    # 合并所有数据
    results = pd.concat(df_list, ignore_index=True)
    # 关闭浏览器释放资源
    browser.quit()
    # 输出结果
    print(results)
    # 可自行添加保存到csv的逻辑：results.to_csv("premier_league_odds.csv", index=False, encoding="utf_8_sig")

你只需要修改base_league_url变量为你需要抓取的任意联赛首页地址，代码会自动遍历该联赛下所有历史赛季、所有分页的赛事数据，无需手动维护URL列表。如果抓取过程中遇到反爬限制，可适当调高time.sleep的延迟时长即可。

内容的提问来源于stack exchange，提问作者user16304089