Selenium中document.scrollingElement失效 无法滚动抓取元素 函数内外运行表现不一致

问题根因 #

• 变量作用域异常：close_up函数中调用的wait对象是main函数内的局部变量，当代码放在main函数外运行时wait为全局变量可正常访问，放入main后变量未传入导致函数运行异常，弹窗关闭逻辑失效，后续滚动、元素读取流程受影响。
• 滚动对象不匹配：站点的推文列表使用独立的滚动容器承载，而非根文档滚动条，直接滚动document.scrollingElement无法触发表格内容加载，视口外的元素调用text属性会返回空值。
• 元素读取逻辑缺陷：提前固定表格行数，分页/滚动后未更新行数，同时未处理懒加载导致的元素未渲染问题。

修复代码 #

调整后的功能函数 #

import time
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import add_data

def close_up(driver, actions, wait):
    time.sleep(1)
    actions.move_to_element(wait.until(EC.element_to_be_clickable((By.XPATH, "//button[@data-dismiss='modal']"))))
    button = driver.find_element(By.XPATH, "//button[@data-dismiss='modal']")
    driver.execute_script("arguments[0].click();", button)
    time.sleep(1)

def check_model_winodows(driver, actions, wait):
    try:
        if len(driver.find_elements(By.XPATH, "(//button[@data-dismiss='modal'])[1]")) > 0:
            close_up(driver, actions, wait)
    except:
        pass
    return driver, actions

调整后的main函数 #

def main(hashtag):
    options = webdriver.ChromeOptions()
    options.add_argument("--disable-infobars")
    options.add_argument("--disable-notifications")
    options.add_argument("--start-maximized")
    options.add_argument("--disable-extensions")
    options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 2})
    options.add_argument('--window-size=1920,1080')
    options.add_experimental_option("prefs", {"profile.default_content_settings.cookies": 2})

    driver = webdriver.Chrome(executable_path='/home/tukaram/chromedriver', options=options)
    driver.maximize_window()
    driver.implicitly_wait(10)
    driver.get("https://www.trackmyhashtag.com/")
    wait = WebDriverWait(driver, 15)

    actions = ActionChains(driver)
    wait.until(EC.visibility_of_element_located((By.ID, "search_keyword"))).send_keys(hashtag, Keys.RETURN)

    check_model_winodows(driver, actions, wait)
    time.sleep(3)
    button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[onclick*='preview-tweets']")))
    driver.execute_script("arguments[0].click();", button)
    check_model_winodows(driver, actions, wait)
    
    rank = 1
    page_number = 2
    total_number_of_pages = 5
    myhashtag = {}
    
    for idx in range(total_number_of_pages):
        print("idx>>>>", idx)
        # 每次分页后重新获取当前页所有行，避免行数变化报错
        elems = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//tbody/tr")))
        for j in range(len(elems)):
            check_model_winodows(driver, actions, wait)
            # 滚动当前行到可见区域
            driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", elems[j])
            time.sleep(0.5)
            
            # 用innerText读取内容，避免不可见元素返回空
            name = driver.execute_script("return arguments[0].querySelector('.tweet-name').innerText;", elems[j])
            print("name>", name)
            myhashtag['user_name'] = name

            userid = driver.execute_script("return arguments[0].querySelector('td').innerText;", elems[j])
            userid = userid.partition('@')[2]
            userid = '@' + userid
            print("userid>", userid)
            myhashtag['user_screen_name'] = userid

            content = driver.execute_script("return arguments[0].querySelector('td:nth-child(2)').innerText;", elems[j])
            print("content", content)
            myhashtag['content'] = content

            date = driver.execute_script("return arguments[0].querySelector('td:nth-child(3)').innerText;", elems[j])
            print("1>>>>", date)
            date = str(date).replace("\n", " ")
            print("2>>>", date)
            date = datetime.strptime(date, '%d %b %Y %H:%M:%S %p')
            print("3>>>", date)
            date = date.strftime('%Y-%m-%dT%H:%M:%SZ')
            print("date", date)
            myhashtag['articleDate'] = date

            engm = driver.execute_script("return arguments[0].querySelector('td:nth-child(4)').innerText;", elems[j])
            print("engagement", engm)
            myhashtag['engagement'] = engm

            impressions = driver.execute_script("return arguments[0].querySelector('td:nth-child(6)').innerText;", elems[j])
            print("impressions", impressions)
            myhashtag['impressions'] = impressions

            myhashtag['rank'] = rank
            rank = rank + 1
            print(myhashtag)
            check_model_winodows(driver, actions, wait)
        
        # 点击下一页
        wait.until(EC.element_to_be_clickable((By.XPATH, f"//a[text()='{page_number}']"))).click()
        page_number = page_number + 1
        print("Page numberrrr", page_number)
        if page_number == 7:
            break
    driver.quit()
    return driver, actions

if __name__ == '__main__':
    for x in add_data.words:
        main(x)

add_data.py 代码保持不变 #

words = ['India','@pakistan'] #words to crawl

核心调整点 #

• 给close_up、check_model_winodows函数新增wait参数，修复变量作用域问题
• 每次处理行时先将目标行滚动到可见区域，替代全局滚动逻辑
• 改用JS读取innerText属性获取元素内容，避免Selenium原生text属性对不可见元素返回空的问题
• 每次分页后重新获取当前页表格行，避免行数变化导致的索引越界问题

内容的提问来源于stack exchange，提问作者pkk