使用Selenium爬取TripAdvisor酒店评论：无法提取星级与入住日期

之前使用针对TripAdvisor景点评论的爬取代码（原URL：https://www.tripadvisor.co.uk/Attraction_Review-g186225-d213774-Reviews-Scudamore_s_Punting_Company-Cambridge_Cambridgeshire_England.html）可以正常运行，但替换为酒店URL（https://www.tripadvisor.com/Hotel_Review-g186405-d215170-Reviews-Portreeves-Arundel_Arun_District_West_Sussex_England.html）后，无法正确提取评论内容、星级评分和入住日期。

修改了分页和评论卡片的选择器后，数据框中的星级评分和评论日期仍为空，尝试提取入住日期时直接导致代码崩溃。修改后的选择器如下：

nxt_pg_sel = 'a.next'
review_sel = 'div[data-test-target="HR_CC_CARD"]'
rev_dets_sel = {
    'from_page': ('', '"staticVal"'),
    'profile_name': 'span>a[href^="\/Profile\/"]',
    'profile_link': ('span>a[href^="\/Profile\/"]', 'href'),
    'about_reviewer': 'span:has(>a[href^="\/Profile\/"])+div',
    'review_votes': 'button[aria-label="Click to add helpful vote"]>span',
    'bubbles': ('svg[aria-label$=" of 5 bubbles"]', 'aria-label'),
    'review_link': ('a[href^="\/ShowUserReviews-"]', 'href'),
    'review_title': 'a[href^="\/ShowUserReviews-"]',
    'about_review': 'div:has(>a[href^="/ShowUserReviews-"])+div:not(:has(div))',
    'review_body': 'div:has(>a[href^="/ShowUserReviews-"])~div>div',
    'review_date': 'div:has(>a[href^="/ShowUserReviews-"])~div:last-child>div',
}

附原景点爬取代码：

##selectforlist
def selectForList(tagSoup, selectors, printList=False):
    if isinstance(selectors, dict):
        return dict(zip(selectors.keys(), selectForList(
            tagSoup, selectors.values(), printList)))
    
    selGen = (( list(sel if isinstance(sel, (tuple, list)) ## generate params
                else [sel])+[None]*2 )[:3] for sel in selectors)
    returnList = [  sel[0] if sel[1] == '"staticVal"' ## [allows placeholders]
                    else selectGet(tagSoup, *sel) for sel in selGen   ]
    
    if printList and not isinstance(printList,str): print(returnList)
    if isinstance(printList,str): print(*returnList, sep=printList)
    return returnList

##original selectors
nxt_pg_sel = 'a[href][data-smoke-attr="pagination-next-arrow"]'
review_sel = 'div[data-automation="reviewCard"]'
rev_dets_sel = {
    'from_page': ('', '"staticVal"'),
    'profile_name': 'span>a[href^="\/Profile\/"]',
    'profile_link': ('span>a[href^="\/Profile\/"]', 'href'),
    'about_reviewer': 'span:has(>a[href^="\/Profile\/"])+div',
    'review_votes': 'button[aria-label="Click to add helpful vote"]>span',
    'bubbles': ('svg[aria-label$=" of 5 bubbles"]', 'aria-label'),
    'review_link': ('a[href^="\/ShowUserReviews-"]', 'href'),
    'review_title': 'a[href^="\/ShowUserReviews-"]',
    'about_review': 'div:has(>a[href^="/ShowUserReviews-"])+div:not(:has(div))',
    'review_body': 'div:has(>a[href^="/ShowUserReviews-"])~div>div',
    'review_date': 'div:has(>a[href^="/ShowUserReviews-"])~div:last-child>div',
}

##set variables
csv_fn_revs = 'Scudamore_s_Punting_Company-tripadvisor_reviews.csv'
csv_fn_pgs = 'Scudamore_s_Punting_Company-tripadvisor_review_pages.csv'
pgNum, maxPages = 0, None
pageUrl = 'https://www.tripadvisor.co.uk/Attraction_Review-g186225-d213774-Reviews-Scudamore_s_Punting_Company-Cambridge_Cambridgeshire_England.html'

##scrape data using web driver
browser = webdriver.Chrome()
browser.maximize_window() # maximize window

reveiws_list, pgList = [], []
while pageUrl and (maxPages is None or pgNum < maxPages):
    pgNum += 1
    pgList.append({'page': pgNum, 'URL': pageUrl})
    try:
        browser.get(pageUrl)
        rev_dets_sel['from_page'] = (pgNum, '"staticVal"')
        pgSoup = BeautifulSoup(browser.page_source, 'html.parser')

        rev_cards = pgSoup.select(review_sel)
        reveiws_list += [selectForList(r, rev_dets_sel) for r in rev_cards]
        pgList[-1]['reviews'] = len(rev_cards)

        next_page = pgSoup.select_one(nxt_pg_sel)
        if next_page:
            pageUrl = 'https://www.tripadvisor.co.uk' + next_page.get('href')
            pgList[-1]['next_page'] = pageUrl
            print('going to', pageUrl)
        else:
            pageUrl = None  # stop condition
    except Exception as e:
        print(f'Stopping on pg{pgNum} due to {type(e)}:\n{e}')
        break

browser.quit() # Close the browser

# Save as csv
pd.DataFrame(reveiws_list).to_csv(csv_fn_revs, index=False)
pd.DataFrame(pgList).to_csv(csv_fn_pgs, index=False)

TripAdvisor酒店页面与景点页面的DOM结构存在差异，需针对性修改选择器，以下是修正后的完整代码及说明：

修正后的核心选择器 #

nxt_pg_sel = 'a[aria-label^="Next page"]'  # 精准定位酒店分页按钮
review_sel = 'div[data-test-target="HR_CC_CARD"]'  # 该选择器无需修改
rev_dets_sel = {
    'from_page': ('', '"staticVal"'),
    'profile_name': 'a[data-test-target="review-author-link"]',
    'profile_link': ('a[data-test-target="review-author-link"]', 'href'),
    'about_reviewer': 'div[data-test-target="review-author-info"]',
    'review_votes': 'button[data-test-target="helpful-button"] span',
    'bubbles': ('div[data-test-target="review-rating"] span', 'aria-label'),  # 星级评分定位
    'review_title': 'div[data-test-target="review-title"]',  # 评论标题
    'review_body': 'div[data-test-target="review-body"]',  # 评论正文
    'review_date': 'div[data-test-target="review-date"]',  # 评论提交日期
    'stay_date': 'div[data-test-target="review-stay-date"]',  # 入住日期
}

完整修正代码 #

from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd

def selectGet(soup, selector, attr=None):
    elem = soup.select_one(selector)
    if not elem:
        return None
    if attr:
        return elem.get(attr)
    return elem.get_text(strip=True)

def selectForList(tagSoup, selectors, printList=False):
    if isinstance(selectors, dict):
        return dict(zip(selectors.keys(), selectForList(
            tagSoup, selectors.values(), printList)))
    
    selGen = (( list(sel if isinstance(sel, (tuple, list))
                else [sel])+[None]*2 )[:3] for sel in selectors)
    returnList = [  sel[0] if sel[1] == '"staticVal"'
                    else selectGet(tagSoup, *sel) for sel in selGen   ]
    
    if printList and not isinstance(printList,str): print(returnList)
    if isinstance(printList,str): print(*returnList, sep=printList)
    return returnList

## 适配酒店的选择器
nxt_pg_sel = 'a[aria-label^="Next page"]'
review_sel = 'div[data-test-target="HR_CC_CARD"]'
rev_dets_sel = {
    'from_page': ('', '"staticVal"'),
    'profile_name': 'a[data-test-target="review-author-link"]',
    'profile_link': ('a[data-test-target="review-author-link"]', 'href'),
    'about_reviewer': 'div[data-test-target="review-author-info"]',
    'review_votes': 'button[data-test-target="helpful-button"] span',
    'bubbles': ('div[data-test-target="review-rating"] span', 'aria-label'),
    'review_title': 'div[data-test-target="review-title"]',
    'review_body': 'div[data-test-target="review-body"]',
    'review_date': 'div[data-test-target="review-date"]',
    'stay_date': 'div[data-test-target="review-stay-date"]',
}

## 设置变量
csv_fn_revs = 'Portreeves-tripadvisor_reviews.csv'
csv_fn_pgs = 'Portreeves-tripadvisor_review_pages.csv'
pgNum, maxPages = 0, None
pageUrl = 'https://www.tripadvisor.com/Hotel_Review-g186405-d215170-Reviews-Portreeves-Arundel_Arun_District_West_Sussex_England.html'

## 启动浏览器爬取
browser = webdriver.Chrome()
browser.maximize_window()

reveiws_list, pgList = [], []
while pageUrl and (maxPages is None or pgNum < maxPages):
    pgNum += 1
    pgList.append({'page': pgNum, 'URL': pageUrl})
    try:
        browser.get(pageUrl)
        # 可选：添加显式等待确保页面加载完成
        # from selenium.webdriver.support.ui import WebDriverWait
        # from selenium.webdriver.support import expected_conditions as EC
        # WebDriverWait(browser, 10).until(EC.presence_of_element_located(('css selector', review_sel)))
        
        rev_dets_sel['from_page'] = (pgNum, '"staticVal"')
        pgSoup = BeautifulSoup(browser.page_source, 'html.parser')

        rev_cards = pgSoup.select(review_sel)
        reveiws_list += [selectForList(r, rev_dets_sel) for r in rev_cards]
        pgList[-1]['reviews'] = len(rev_cards)

        next_page = pgSoup.select_one(nxt_pg_sel)
        if next_page:
            pageUrl = 'https://www.tripadvisor.com' + next_page.get('href')
            pgList[-1]['next_page'] = pageUrl
            print('跳转至:', pageUrl)
        else:
            pageUrl = None
    except Exception as e:
        print(f'第{pgNum}页出错，停止爬取：{type(e)}\n{e}')
        break

browser.quit()

# 保存数据
pd.DataFrame(reveiws_list).to_csv(csv_fn_revs, index=False)
pd.DataFrame(pgList).to_csv(csv_fn_pgs, index=False)

关键调整说明 #

1. 星级评分：酒店页面的星级不再使用svg标签，改为定位div[data-test-target="review-rating"]下的span，通过aria-label获取评分文本
2. 入住日期：直接使用div[data-test-target="review-stay-date"]定位，避免了之前层级选择器错误导致的崩溃
3. 评论内容：全部使用TripAdvisor官方的data-test-target属性定位，比依赖DOM层级更稳定，不易受页面结构变动影响
4. 分页按钮：用aria-label属性精准定位下一页，替代模糊的a.next选择器
5. 作者信息：同样使用data-test-target属性定位，适配酒店页面的作者信息布局

内容的提问来源于stack exchange，提问作者speeddefiesgravity