爬取Trustpilot评论存Pandas DataFrame遇AttributeError怎么解决？

问题场景 #

使用以下Python代码爬取Trustpilot前20页评论并存储到Pandas DataFrame：

from bs4 import BeautifulSoup
import requests
import pandas as pd
import datetime as dt

# Initialize lists
review_titles = []
review_dates_original = []
review_dates = []
review_ratings = []
review_texts = []
page_number = []

# Set Trustpilot page numbers to scrape here
from_page = 1
to_page = 20

for i in range(from_page, to_page + 1):
   response = requests.get(f"https://uk.trustpilot.com/review/www.abc.com?page={i}")
web_page = response.text
soup = BeautifulSoup(web_page, "html.parser")

for review in soup.find_all(class_ = "paper_paper__1PY90 paper_square__lJX8a card_card__lQWDv card_noPadding__D8PcU card_square___tXn9 styles_navigationContainer__kPGA_"):
    # Review titles                  
    review_title = review.find(class_ = "typography_heading-s__f7029 typography_appearance-default__AAY17")
    review_titles.append(review_title.getText()) 
    # Review dates
    review_date_original = review.select_one(selector="time")
    review_dates_original.append(review_date_original.getText())

    # Convert review date texts into Python datetime objects
    review_date = review.select_one(selector="time").getText().replace("Updated ", "")
    if "hours ago" in review_date.lower() or "hour ago" in review_date.lower():
        review_date = dt.datetime.now().date()
    elif "a day ago" in review_date.lower():
        review_date = dt.datetime.now().date() - dt.timedelta(days=1)
    elif "days ago" in review_date.lower():
        review_date = dt.datetime.now().date() - dt.timedelta(days=int(review_date[0]))
    else:
        review_date = dt.datetime.strptime(review_date, "%b %d, %Y").date()
    review_dates.append(review_date)

    # Review ratings
    review_rating = review.find(class_ = "star-rating_starRating__4rrcf star-rating_medium__iN6Ty").findChild()
    review_ratings.append(review_rating["alt"])
    
    # When there is no review text, append "" instead of skipping so that data remains in sequence with other review data e.g. review_title
    review_text = review.find(class_ = "typography_body-l__KUYFJ typography_appearance-default__AAY17 typography_color-black__5LYEn")
    if review_text == None:
        review_texts.append("")
    else:
        review_texts.append(review_text.getText())
    
    # Trustpilot page number
    page_number.append(i)

 # Create final dataframe from lists
df_reviews = pd.DataFrame(list(zip(review_titles, review_dates_original, review_dates, 
review_ratings, review_texts, page_number)),
            columns =['review_title', 'review_date_original', 'review_date', 
'review_rating', 'review_text', 'page_number'])

错误信息 #

运行代码时触发以下错误：

AttributeError                            Traceback (most recent call last)
 in 
   24         # Review titles
   25         review_title = review.find(class_ = "typography_heading-s__f7029 
   typography_appearance-default__AAY17")
 ---> 26         review_titles.append(review_title.getText())
    27         # Review dates
  28         review_date_original = review.select_one(selector="time")

AttributeError: 'NoneType' object has no attribute 'getText'

问题原因 #

部分评论的标题、日期、评分等元素查找结果为None，直接调用.getText()或访问属性会触发AttributeError，因为None对象不具备这些方法和属性；同时原代码存在循环缩进错误，导致仅处理最后一页数据。

解决方法 #

对每个可能返回None的元素添加非空判断，修复循环缩进问题，并增加异常处理确保日期转换的鲁棒性。修改后的完整代码如下：

from bs4 import BeautifulSoup
import requests
import pandas as pd
import datetime as dt

# Initialize lists
review_titles = []
review_dates_original = []
review_dates = []
review_ratings = []
review_texts = []
page_number = []

# Set Trustpilot page numbers to scrape here
from_page = 1
to_page = 20

for i in range(from_page, to_page + 1):
    response = requests.get(f"https://uk.trustpilot.com/review/www.abc.com?page={i}")
    web_page = response.text
    soup = BeautifulSoup(web_page, "html.parser")

    for review in soup.find_all(class_ = "paper_paper__1PY90 paper_square__lJX8a card_card__lQWDv card_noPadding__D8PcU card_square___tXn9 styles_navigationContainer__kPGA_"):
        # Review titles
        review_title = review.find(class_ = "typography_heading-s__f7029 typography_appearance-default__AAY17")
        review_titles.append(review_title.getText() if review_title else "")
        
        # Review dates (original)
        review_date_original = review.select_one("time")
        original_date_text = review_date_original.getText() if review_date_original else ""
        review_dates_original.append(original_date_text)
        
        # Convert review date texts into Python datetime objects
        review_date = None
        if review_date_original:
            raw_date = original_date_text.replace("Updated ", "")
            if "hours ago" in raw_date.lower() or "hour ago" in raw_date.lower():
                review_date = dt.datetime.now().date()
            elif "a day ago" in raw_date.lower():
                review_date = dt.datetime.now().date() - dt.timedelta(days=1)
            elif "days ago" in raw_date.lower():
                # 处理多位数天数，比如"10 days ago"
                try:
                    days = int(raw_date.split()[0])
                    review_date = dt.datetime.now().date() - dt.timedelta(days=days)
                except ValueError:
                    review_date = None
            else:
                try:
                    review_date = dt.datetime.strptime(raw_date, "%b %d, %Y").date()
                except ValueError:
                    review_date = None
        review_dates.append(review_date)
        
        # Review ratings
        rating_container = review.find(class_ = "star-rating_starRating__4rrcf star-rating_medium__iN6Ty")
        review_rating = rating_container.findChild()["alt"] if rating_container else ""
        review_ratings.append(review_rating)
        
        # Review texts
        review_text = review.find(class_ = "typography_body-l__KUYFJ typography_appearance-default__AAY17 typography_color-black__5LYEn")
        review_texts.append(review_text.getText() if review_text else "")
        
        # Trustpilot page number
        page_number.append(i)

# Create final dataframe from lists
df_reviews = pd.DataFrame(
    list(zip(review_titles, review_dates_original, review_dates, review_ratings, review_texts, page_number)),
    columns=['review_title', 'review_date_original', 'review_date', 'review_rating', 'review_text', 'page_number']
)

关键修改点 #

• 对所有可能返回None的元素添加非空判断，避免直接调用方法
• 日期转换部分增加try-except块，处理格式异常和多位数天数的情况
• 修复原代码的循环缩进错误，确保遍历所有页面

内容的提问来源于stack exchange，提问作者Django0602