Python Selenium Firefox驱动无法滚动到底部，如何强制修复？

原有滚动代码（包括execute_script直接滚底、DOWN/PageDown按键操作）均失效，执行时页面滚动一次后强制回到顶部，代码因页面高度未变化误认为已到达底部，3周前该代码可正常运行。

原有代码：

def scroll_to_bottom(): 
    '''
    Scroll down the page the whole way. This must be done to show all images on page to download.
    '''

    print("[+] Starting scroll down of page. This needs to be performed to view all images.")
    count = 0
    start_time = time.perf_counter()
    try:
        last_height = driver.execute_script("return document.body.scrollHeight")
        while True:
            if count % 10 == 0: 
                elapsed_time = time.perf_counter() - start_time
                elapsed_time_clean = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
                print("[i] Still scrolling, its been " + str(elapsed_time_clean) + ", please continue to standby... C: " + str(count))
            time.sleep(3)
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(3)

            # Check if hight has changed
            new_height = driver.execute_script("return document.body.scrollHeight")
            vprint("last_height: " + str(last_height) + ", new_height: " + str(new_height))
            if new_height == last_height:
                break
            last_height = new_height
            count += 1

    except Exception as exception:
        print('[!] Exception exception: '+str(exception))
        pass

    print("[i] Scroll down of whole page complete.")
    return

1. 渐进式滚动模拟人类行为 #

避免一次性跳转到页面底部，改成每次滚动固定距离（比如窗口高度的1.5倍），降低被反爬检测的概率：

def scroll_to_bottom():
    print("[+] Starting progressive scroll down of page.")
    count = 0
    start_time = time.perf_counter()
    try:
        window_height = driver.execute_script("return window.innerHeight")
        total_scrolled = 0
        while True:
            if count % 20 == 0:
                elapsed_time = time.perf_counter() - start_time
                elapsed_time_clean = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
                print(f"[i] Still scrolling, elapsed time: {elapsed_time_clean}, scroll count: {count}")
            # 每次滚动窗口高度的1.5倍
            driver.execute_script(f"window.scrollBy(0, {window_height * 1.5});")
            count += 1
            total_scrolled += window_height * 1.5
            time.sleep(2)
            
            current_height = driver.execute_script("return document.body.scrollHeight")
            # 判断是否接近底部（差200px以内）
            if total_scrolled >= current_height - 200:
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(1)
                new_height = driver.execute_script("return document.body.scrollHeight")
                if new_height == current_height:
                    break
                else:
                    current_height = new_height
                    total_scrolled = current_height - window_height
    except Exception as exception:
        print(f'[!] Exception: {str(exception)}')
        pass
    print("[i] Scroll down complete.")
    return

2. 用ActionChains模拟鼠标滚轮滚动 #

直接调用浏览器原生鼠标滚轮事件，比scrollTo更难被检测：

from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.actions.wheel_input import ScrollOrigin

def scroll_to_bottom():
    print("[+] Starting mouse wheel scroll down.")
    count = 0
    start_time = time.perf_counter()
    try:
        last_height = driver.execute_script("return document.body.scrollHeight")
        while True:
            if count % 10 == 0:
                elapsed_time = time.perf_counter() - start_time
                elapsed_time_clean = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
                print(f"[i] Still scrolling, elapsed time: {elapsed_time_clean}, scroll count: {count}")
            # 以body元素为滚动原点，向下滚动1000单位
            scroll_origin = ScrollOrigin.from_element(driver.find_element("tag name", "body"))
            ActionChains(driver)\
                .scroll_from_origin(scroll_origin, 0, 1000)\
                .perform()
            count += 1
            time.sleep(2)
            
            new_height = driver.execute_script("return document.body.scrollHeight")
            if new_height == last_height:
                # 多滚动几次确认是否真的到底
                for _ in range(3):
                    ActionChains(driver)\
                        .scroll_from_origin(scroll_origin, 0, 500)\
                        .perform()
                    time.sleep(1)
                new_height = driver.execute_script("return document.body.scrollHeight")
                if new_height == last_height:
                    break
            last_height = new_height
    except Exception as exception:
        print(f'[!] Exception: {str(exception)}')
        pass
    print("[i] Scroll down complete.")
    return

3. 隐藏Selenium自动化特征 #

页面可能通过浏览器特征检测自动化工具，给Firefox驱动添加参数隐藏这些特征：

from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service

options = Options()
# 禁用自动化控制提示
options.add_argument("--disable-blink-features=AutomationControlled")
# 模拟真实用户代理
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0")
# 可选：启用无头模式（无界面运行）
# options.add_argument("--headless")

# 初始化驱动，替换为你的geckodriver路径
driver = webdriver.Firefox(service=Service("/path/to/geckodriver"), options=options)

4. 结合元素加载状态判断滚动终点 #

不要仅依赖页面高度，结合目标元素（比如未加载的图片）的状态判断是否需要继续滚动：

def scroll_to_bottom():
    print("[+] Starting scroll with element load check.")
    count = 0
    start_time = time.perf_counter()
    try:
        while True:
            if count % 10 == 0:
                elapsed_time = time.perf_counter() - start_time
                elapsed_time_clean = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
                print(f"[i] Still scrolling, elapsed time: {elapsed_time_clean}, scroll count: {count}")
            # 滚动当前窗口高度
            driver.execute_script("window.scrollBy(0, window.innerHeight);")
            count += 1
            time.sleep(2)
            
            # 检查是否有待加载的图片（替换为你页面的图片选择器）
            pending_images = driver.find_elements("css selector", "img[data-src]")
            if not pending_images:
                # 再滚动3次确认
                for _ in range(3):
                    driver.execute_script("window.scrollBy(0, window.innerHeight);")
                    time.sleep(1)
                pending_images = driver.find_elements("css selector", "img[data-src]")
                if not pending_images:
                    break
    except Exception as exception:
        print(f'[!] Exception: {str(exception)}')
        pass
    print("[i] Scroll down complete.")
    return

内容的提问来源于stack exchange，提问作者Dave