Python Selenium Webdriver爬取谷歌餐厅评论无法加载全部数据问题求助

• 核心错误在循环计数逻辑：原代码没有将total_reviews更新为实际加载到的评论数量，而是手动执行total_reviews +=1，计数和实际加载量完全脱节，自然无法加载超过初始的50条
• 谷歌评论的懒加载是绑定在评论列表的独立滚动容器上的，直接滚动最后一条评论的触发概率低，经常无法触发加载请求
• 没有异常跳出机制，遇到加载失败的场景会进入无限循环

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

driver = webdriver.Chrome()
base_url = 'https://www.google.com/search?tbs=lf:1,lf_ui:9&tbm=lcl&sxsrf=AOaemvJFjYToqQmQGGnZUovsXC1CObNK1g:1633336974491&q=10+famous+restaurants+in+Dunedin&rflfq=1&num=10&sa=X&ved=2ahUKEwiTsqaxrrDzAhXe4zgGHZPODcoQjGp6BAgKEGo&biw=1280&bih=557&dpr=2#lrd=0xa82eac0dc8bdbb4b:0x4fc9070ad0f2ac70,1,,,&rlfi=hd:;si:5749134142351780976,l,CiAxMCBmYW1vdXMgcmVzdGF1cmFudHMgaW4gRHVuZWRpbiJDUjEvZ2VvL3R5cGUvZXN0YWJsaXNobWVudF9wb2kvcG9wdWxhcl93aXRoX3RvdXJpc3Rz2gENCgcI5Q8QChgFEgIIFkiDlJ7y7YCAgAhaMhAAEAEQAhgCGAQiIDEwIGZhbW91cyByZXN0YXVyYW50cyBpbiBkdW5lZGluKgQIAxACkgESaXRhbGlhbl9yZXN0YXVyYW50mgEkQ2hkRFNVaE5NRzluUzBWSlEwRm5TVU56ZW5WaFVsOUJSUkFCqgEMEAEqCCIEZm9vZCgA,y,2qOYUvKQ1C8;mv:[[-45.8349553,170.6616387],[-45.9156414,170.4803685]]'
driver.get(base_url)
WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,"//div[./span[text()='Newest']]"))).click()

title = driver.find_element(By.XPATH,"//div[@class='P5Bobd']").text
address = driver.find_element(By.XPATH,"//div[@class='T6pBCe']").text
overall_rating = driver.find_element(By.XPATH,"//div[@class='review-score-container']//span[@class='Aq14fc']").text
total_reviews_text =driver.find_element(By.XPATH,"//div[@class='review-score-container']//div//div//span//span[@class='z5jxId']").text
 
num_reviews = int(total_reviews_text.split()[0])
# 定位评论滚动容器
review_scroll_container = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, 'div:has(> div.gws-localreviews__google-review)'))
)
all_reviews = WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div.gws-localreviews__google-review')))
total_reviews = len(all_reviews)
# 增加重复计数，避免死循环
repeat_count = 0
max_repeat = 5

while total_reviews < num_reviews and repeat_count < max_repeat:
    # 滚动评论容器到底部
    driver.execute_script('arguments[0].scrollTop = arguments[0].scrollHeight', review_scroll_container)
    # 等待加载指示器消失
    try:
        WebDriverWait(driver, 5, 0.25).until_not(EC.presence_of_element_located((By.CSS_SELECTOR, 'div[class$="activityIndicator"]')))
    except:
        pass
    # 等待0.5秒保证内容渲染
    time.sleep(0.5)
    # 获取新的评论列表
    new_reviews = WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div.gws-localreviews__google-review')))
    new_total = len(new_reviews)
    print(f"当前加载到评论数：{new_total}")
    if new_total == total_reviews:
        repeat_count +=1
    else:
        repeat_count =0
        total_reviews = new_total
        all_reviews = new_reviews

# 后续可自行添加提取评论内容的逻辑
print(f"最终加载到评论数：{total_reviews}")

• 修正了计数逻辑：每次加载后用实际获取到的评论列表长度更新计数，不再手动累加
• 改为滚动评论列表专属容器，更符合谷歌评论的懒加载触发逻辑
• 增加了重复加载判断，连续5次加载没有新评论就自动退出，避免死循环
• 替换了已经废弃的find_element_by_*写法，适配Selenium 4+版本

内容的提问来源于stack exchange，提问作者user2293224