IMDB用户评分分页爬虫问题：重复爬取第一页数据

我正在开发一款IMDB用户评分Web Scraper，目标是将用户评分中的部分数据保存至Excel（后续计划实现电影推荐功能）。每页显示100部影片，当用户评分的影片数量超过100部时会出现分页。我编写了循环逻辑，先保存第一页的100部影片，然后切换至下一页，但在第二页（影片101-200）时，Python仍会将第一页（1-100）的影片保存至Excel，更多页数时问题依旧。

import requests
from bs4 import BeautifulSoup
import pandas as pd
import openpyxl
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
from PIL import Image
import math
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

url = "https://www.imdb.com/user/ur163997524/ratings"
response = requests.get(url)
soup = BeautifulSoup(response.content, "html.parser")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
page = driver.get(url)
def nextButtonIsVisible():
    buttons = driver.find_elements(By.CLASS_NAME, 'lister-page-next')
    return len(buttons) > 0

def clickNextPage():
    print("Click NEXT button")
    driver.find_element(By.CLASS_NAME, 'lister-page-next').click()

movies = soup.find_all("div", class_="lister-item-content")
movie_names = []
movie_actors = []
movie_years = []
user_rank = []
imdb_rank = []
movie_genre = []
pagnitation_range = ""

upperPages = driver.find_element(By.XPATH, "//span[contains(@id, 'lister-header-current-size')]").text
#print(upperPages)
moviesQuantity = int(upperPages)
pagesQuantity = moviesQuantity/100
pageCounter = math.ceil(pagesQuantity)
upperPages = int(upperPages)

movies = soup.find_all("div", class_="lister-item-content")
while nextButtonIsVisible():
    for movie in movies:
        #movies = soup.find_all("div", class_="lister-item-content")
        name = movie.find("a").text
        year = movie.find("span", class_="lister-item-year").text.strip("()")

        empty = ""
        for char in year:
            if char.isdigit():
                remaining_year = year[len(empty):]
                break
            empty += char

        actors = movie.find_all("a")[1:]
        actors = [actor.text for actor in actors]
        actors = actors[11:]
        imdbrank = movie.find('span', class_="ipl-rating-star__rating").get_text().strip()
        userRank = movie.find('div', class_="ipl-rating-star ipl-rating-star--other-user small").get_text().strip()
        genre = movie.find('span', class_="genre").get_text().strip()

        movie_names.append(name)
        movie_years.append(remaining_year)
        user_rank.append(userRank)
        imdb_rank.append(imdbrank)
        movie_genre.append(genre)
        movie_actors.append(", ".join(actors))
        data = {"Name": movie_names, "Genre": movie_genre, "Year": movie_years, "User rank": user_rank,
                "IMDB rank": imdb_rank, "Actors": movie_actors}
        print(movies)
        df = pd.DataFrame(data)
        df.to_excel("movie_ratings.xlsx", index=False)
    def nextButtonIsVisible():
        buttons = driver.find_elements(By.CLASS_NAME, 'lister-page-next')
        return len(buttons) > 0
    if nextButtonIsVisible():
        clickNextPage()
    else:
        break
df.to_excel("movie_ratings.xlsx", index=False)

已尝试的解决方案：

• 在while循环内重新声明movies = soup.find_all("div", class_="lister-item-content")
• 爬取前刷新页面

内容的提问来源于stack exchange，提问作者szymanskimilosz