Python Selenium Headless Chrome无法滚动加载Facebook评论问题求助

问题核心 #

Headless模式下Chrome的自动化特征易被Facebook反爬机制识别，且原代码针对整个页面body滚动，而Facebook评论实际加载在独立的滚动容器内，导致滚动无法触发评论加载逻辑。

修复方案 #

1. 增强浏览器伪装，规避反爬检测 #

添加更多Chrome启动参数，模拟真实浏览器环境：

• --disable-blink-features=AutomationControlled：禁用自动化特征检测
• --enable-javascript：确保JavaScript正常运行
• --lang=en-US,en：设置标准语言环境
• 调整窗口尺寸为更贴近真实设备的1920x1080

2. 定位评论区专属滚动容器，针对性触发加载 #

Facebook评论列表存在于特定的可滚动容器中，直接滚动该容器才能触发评论加载逻辑，而非整个页面body。

修改后的完整代码 #

import datetime
import re
import time

from decouple import config
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
import yake
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException

# 更新为较新版本的Chrome用户代理
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"

options = Options()
# 基础运行配置
options.add_argument('--disable-gpu-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--disable-software-rasterizer')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--no-sandbox')
# 增强反爬伪装
options.add_argument("--window-size=1920,1080")
options.add_argument("--headless=new")
options.add_argument(f"--user-agent={user_agent}")
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--enable-javascript')
options.add_argument('--lang=en-US,en')

driver = webdriver.Chrome(options=options)

# 登录流程优化
driver.get("https://www.facebook.com/")
try:
    email_input = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "email"))
    )
    password_input = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "pass"))
    )

    email_input.send_keys(config("FB_EMAIL_INPUT"))
    password_input.send_keys(config("FB_PASSWORD_INPUT"))
    password_input.send_keys(Keys.RETURN)

    # 验证登录状态
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, "//div[@aria-label='Your profile']"))
    )
    print("Login successful")
except (NoSuchElementException, TimeoutException):
    print("Login failed")
    driver.quit()
    exit()

POST_URL = "https://www.facebook.com/thebetterindia/posts/pfbid025Yo2f5Qsd8NDL4AoFoHuvjeAURiRVc7rQ4uZBbULMuUWCfZ9NURRfeVha7aPpnn3l"
driver.get(POST_URL)

def infinite_scroll(driver, timeout=2):
    try:
        # 等待评论区容器加载完成
        comment_container = WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.XPATH, "//div[@role='feed']//div[contains(@class, 'x1yztbdb')]"))
        )
        last_height = driver.execute_script("return arguments[0].scrollHeight;", comment_container)
        
        while True:
            # 滚动评论容器到底部
            driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight;", comment_container)
            time.sleep(timeout)
            new_height = driver.execute_script("return arguments[0].scrollHeight;", comment_container)
            
            # 处理"查看更多评论"按钮
            try:
                load_more_btn = WebDriverWait(driver, 3).until(
                    EC.element_to_be_clickable((By.XPATH, "//div[contains(text(), 'View more comments')]"))
                )
                load_more_btn.click()
                time.sleep(timeout)
                new_height = driver.execute_script("return arguments[0].scrollHeight;", comment_container)
            except TimeoutException:
                pass
            
            if new_height == last_height:
                break
            last_height = new_height
    except Exception as e:
        print(f"Scroll error: {e}")

try:
    infinite_scroll(driver, timeout=2)
    print("Scroll completed")
except Exception as e:
    print(f"An exception occurred: {e}")
finally:
    driver.quit()

关键修改说明 #

• 更新用户代理为较新版本，避免因UA过旧被检测
• 添加--disable-blink-features=AutomationControlled参数，防止Facebook识别Selenium自动化
• 替换页面body滚动为评论区容器滚动，精准触发加载逻辑
• 增加"查看更多评论"按钮的自动点击，处理Facebook分段加载机制

内容的提问来源于stack exchange，提问作者silverFoxA