Selenium点击Show More按钮遇阻及GraphQL爬取报错求助

安全阿岳

2026-6-17

问题描述

最新更新见底部

我使用Python 3.12.3和Selenium进行网页爬取，希望在爬取前加载更多书评内容（理想情况是加载全部或尽可能多，网站可能会限制单页总评论数）。手动点击Show More按钮至少10次都能成功，但自动化点击始终失败。

目标按钮：Show More Button
我无法成功点击该按钮，推测原因是其包含:before伪元素，且页面存在多个div.Button__container元素，导致点击时误触其他按钮。

爬取书评脚本

以下是我用于爬取书评的脚本，但它无法实现点击加载更多的功能，我希望通过Selenium解决这个问题：

import requests
import os
from bs4 import BeautifulSoup
import pandas as pd
import time
from selenium import webdriver
from IPython.display import display, Image

driver = webdriver.Chrome()
title = driver.title

for i in range(0,1000,100):
    # McCurdy - 《我很高兴我妈死了》
    response = requests.get('https://www.goodreads.com/book/show/26074156/reviews?reviewFilters={%22workId%22:%22kca://work/amzn1.gr.work.v1.FSsY8ohzUZCeEXoBsiEYqw%22,%22after%22:%22NjgxNSwxNTAwNjU3MjE4NDI1%22}')
print(response.status_code)

time.sleep(6)

doc = BeautifulSoup(response.text, 'html.parser')

df = pd.DataFrame(columns=['ReviewDate','ReviewerName','Rating','ReviewText','ReviewerMeta'])

ratings = []

# 遍历所有找到的元素
for tag in book_tags_:
    # 获取当前元素的aria-label属性
    aria_label = tag.get('aria-label')
    
    # 检查aria-label是否不为空且符合预期格式
    if aria_label and 'Rating ' in aria_label and ' out of 5' in aria_label:
        # 拆分aria-label提取所需文本
        rating_text = aria_label.split('Rating ')[1].split(' out of 5')[0]
        
        # 将评分文本添加到列表
        ratings.append(rating_text)
    else:
        print(f"Skipping element with aria-label: {aria_label}")

# 从评分列表创建DataFrame
df = pd.DataFrame({'Rating': ratings})

# 其他字段的提取逻辑与此类似

尝试点击Show More按钮的代码

以下是我尝试点击按钮的代码：

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException, ElementClickInterceptedException

# 初始化WebDriver
driver = webdriver.Chrome()  # 或使用你其他的驱动

url = "https://www.goodreads.com/book/show/59364173/reviews?reviewFilters={%22workId%22:%22kca://work/amzn1.gr.work.v3.JeHZlXvg2e1mD9_k%22,%22after%22:%22MjYwMTYsMTY2MDc1MjY5MjY2Mw%22}"

# 打开页面
driver.get(url)
time.sleep(20)

def click_show_more():
    while True:
        try:
            # 滚动到页面底部确保按钮可见
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            
            # 找到'Show More'按钮
            show_more_button = WebDriverWait(driver, 10).until(
                EC.visibility_of_element_located((By.CSS_SELECTOR, 'div.Divider.Divider--contents.Divider--largeMargin > div.Button__container'))
            )
            
            # 将元素滚动到视图中
            driver.execute_script("arguments[0].scrollIntoView(true);", show_more_button)
            
            # 确保元素可点击
            WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div.Divider.Divider--contents.Divider--largeMargin > div.Button__container')))
            
            # 必要时使用JavaScript点击按钮
            driver.execute_script("arguments[0].click();", show_more_button)
            
            # 点击后等待新内容加载（可选）
            WebDriverWait(driver, 10).until(EC.staleness_of(show_more_button))
        
        except TimeoutException:
            print("未找到更多'Show More'按钮或超时。")
            break
        except StaleElementReferenceException:
            print("StaleElementReferenceException: 重新查找按钮。")
            continue
        except ElementClickInterceptedException:
            print("ElementClickInterceptedException: 元素被遮挡。")
            continue
        except Exception as e:
            print(f"点击'Show More'按钮出错: {e}")
            break

# 调用函数
click_show_more()

更新后的问题与代码

根据@x1337loser的建议，我在get_data()函数中添加了列表追加逻辑，之后出现报错（报错截图）。我还移除了打印输出中的用户头像，并在脚本末尾添加了数据字典和写入CSV的逻辑，这些步骤未引发问题。

以下是更新后的代码：

import time
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from bs4 import BeautifulSoup
import pandas as pd

ReviewText = []
ratings = []
Reviewer = []
reviewdt = []
ReviewerId = []
Author_ID = []

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def get_data(content):
    data = content['data']['getReviews']['edges']
    for i in data:
        id_user = i['node']['creator']['id']
        rev_img = i['node']['creator']['imageUrlSquare']
        is_author = i['node']['creator']['isAuthor']
        follower_count = i['node']['creator']['followersCount']
        name = i['node']['creator']['name']
        review = BeautifulSoup(i['node']['text'], "lxml").text # 移除文本中的HTML标签
        review_create_data = i['node']['createdAt']
        result_ms = pd.to_datetime(review_create_data,unit='ms') # 解码时间戳
        review_liked = i['node']['likeCount']
        rating = i['node']['rating']
                                                                                                                                   # 可在result_ms和reviewer_follower之间添加\nreviewer_image:  {rev_img}
        print(f"评论者姓名:  {name}\n评论者用户ID:  {id_user}\n是否为作者:  {is_author}\n评论日期:  {result_ms}\n评论者粉丝数:  {follower_count}\n评论内容:  {review}\n评分:  {rating}\n点赞数:  {review_liked}\n=========================================")

        # print(f"是否为作者:  {is_author}\n评论者粉丝数:  {follower_count}\n点赞数:  {review_liked}\n=========================================")


        ReviewText_content = {review}
        rating_text = {rating}
        Reviewer_name = {name}
        reviewdt_content = {result_ms}
        reviewer_id = {id_user}
        author = {is_author}
        
        # 将内容添加到对应列表
        Reviewer.append(Reviewer_name)
        reviewdt.append(reviewdt_content)
        ratings.append(rating_text)    
        ReviewText.append(ReviewText_content)
        ReviewerId.append(reviewer_id)
        Author_ID.append(author)


def gatherNextPage(resourceId):
    url = "https://kxbwmqov6jgg3daaamb744ycu4.appsync-api.us-east-1.amazonaws.com/graphql"
    headers = {
        "User-Agent":"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0",
        "X-Api-Key": "da2-xpgsdydkbregjhpr6ejzqdhuwy" # 服务器自动添加的API密钥，客户端请求会严格校验
    }
    data = {
    "operationName": "getReviews",
    "variables": {
        "filters": {
        "resourceType": "WORK",
        "resourceId": f"{resourceId}"
        },
        "pagination": {
        "limit": 100
        }
    },
    "query": "query getReviews($filters: BookReviewsFilterInput!, $pagination: PaginationInput) {\n  getReviews(filters: $filters, pagination: $pagination) {\n    ...BookReviewsFragment\n    __typename\n  }\n}\n\nfragment BookReviewsFragment on BookReviewsConnection {\n  totalCount\n  edges {\n    node {\n      ...ReviewCardFragment\n      __typename\n    }\n    __typename\n  }\n  pageInfo {\n    prevPageToken\n    nextPageToken\n    __typename\n  }\n  __typename\n}\n\nfragment ReviewCardFragment on Review {\n  __typename\n  id\n  creator {\n    ...ReviewerProfileFragment\n    __typename\n  }\n  recommendFor\n  updatedAt\n  createdAt\n  spoilerStatus\n  lastRevisionAt\n  text\n  rating\n  shelving {\n    shelf {\n      name\n      webUrl\n      __typename\n    }\n    taggings {\n      tag {\n        name\n        webUrl\n        __typename\n      }\n      __typename\n    }\n    webUrl\n    __typename\n  }\n  likeCount\n  viewerHasLiked\n  commentCount\n}\n\nfragment ReviewerProfileFragment on User {\n  id: legacyId\n  imageUrlSquare\n  isAuthor\n  ...SocialUserFragment\n  textReviewsCount\n  viewerRelationshipStatus {\n    isBlockedByViewer\n    __typename\n  }\n  name\n  webUrl\n  contributor {\n    id\n    works {\n      totalCount\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n\nfragment SocialUserFragment on User {\n  viewerRelationshipStatus {\n    isFollowing\n    isFriend\n    __typename\n  }\n  followersCount\n  __typename\n}\n"
    }
    n, data_next = 1, data
    while True:
        time.sleep(3)
        resp = requests.post(url, headers=headers, json=data_next, verify=False)
        data = resp.json()
        get_data(data)
        nextPageToken = data['data']['getReviews']['pageInfo']['nextPageToken']
        if not nextPageToken:
            break

        data_next = {
        "operationName": "getReviews",
        "variables": {
            "filters": {
            "resourceType": "WORK",
            "resourceId": f"{resourceId}"
            },
            "pagination": {
            "after": f"{nextPageToken}", # 从响应数据获取nextPageToken，每次请求最多返回100条结果
            "limit": 100
            }
        },
        "query": "query getReviews($filters: BookReviewsFilterInput!, $pagination: PaginationInput) {\n  getReviews(filters: $filters, pagination: $pagination) {\n    ...BookReviewsFragment\n    __typename\n  }\n}\n\nfragment BookReviewsFragment on BookReviewsConnection {\n  totalCount\n  edges {\n    node {\n      ...ReviewCardFragment\n      __typename\n    }\n    __typename\n  }\n  pageInfo {\n    prevPageToken\n    nextPageToken\n    __typename\n  }\n  __typename\n}\n\nfragment ReviewCardFragment on Review {\n  __typename\n  id\n  creator {\n    ...ReviewerProfileFragment\n    __typename\n  }\n  recommendFor\n  updatedAt\n  createdAt\n  spoilerStatus\n  lastRevisionAt\n  text\n  rating\n  shelving {\n    shelf {\n      name\n      webUrl\n      __typename\n    }\n    taggings {\n      tag {\n        name\n        webUrl\n        __typename\n      }\n      __typename\n    }\n    webUrl\n    __typename\n  }\n  likeCount\n  viewerHasLiked\n  commentCount\n}\n\nfragment ReviewerProfileFragment on User {\n  id: legacyId\n  imageUrlSquare\n  isAuthor\n  ...SocialUserFragment\n  textReviewsCount\n  viewerRelationshipStatus {\n    isBlockedByViewer\n    __typename\n  }\n  name\n  webUrl\n  contributor {\n    id\n    works {\n      totalCount\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n\nfragment SocialUserFragment on User {\n  viewerRelationshipStatus {\n    isFollowing\n    isFriend\n    __typename\n  }\n  followersCount\n  __typename\n}\n"
        }

gatherNextPage("kca://work/amzn1.gr.work.v3.JeHZlXvg2e1mD9_k")

data_dict = {
    'ReviewDate': reviewdt,
    'ReviewerName': Reviewer,
    'Rating': ratings,
    'ReviewText': ReviewText,
    'ReviewerID': ReviewerId,
    'IsAuthor': Author_ID

    # 'ReviewerMeta': reviewer_meta
}



df = pd.DataFrame(data_dict)
df.to_csv('McCurdy_Sample_Sept30.csv', index=False)

内容的提问来源于stack exchange，提问作者Bad_coder

Selenium点击Show More按钮遇阻及GraphQL爬取报错求助

爬取书评脚本 #

尝试点击Show More按钮的代码 #

更新后的问题与代码 #

爬取书评脚本

尝试点击Show More按钮的代码

更新后的问题与代码