Selenium获取Curiosity Stream页面动态标题的等待问题

问题背景 #

使用Selenium根据页面标题对媒体文件重命名归类时，遇到Curiosity Stream页面后台加载的问题：点击纪录片系列非第一集时，页面初始返回单集名称，后台加载完成后标题才会更新为系列标题，现有等待方案无法稳定捕获最终状态，且不想依赖不稳定的time.sleep。

已尝试的方案 #

1. 直接等待标题变化 #

该方案仅等待标题首次变化，无法确保后续不再更新，导致仍可能获取到单集标题：

import selenium.webdriver.support.expected_conditions as ec
from selenium.webdriver.support.wait import WebDriverWait
browser = Firefox()
wait = WebDriverWait(driver = browser, timeout = 30)
browser.get('https://curiositystream.com/video/3558')
wait.until_not(ec.title_is(current_title))

2. 等待特定元素可见 #

通过等待显示标题的元素可见，降低了问题出现频率，但仍无法彻底解决：

print(wait.until(
    ec.visibility_of_element_located((
        'xpath',
        '//button[@aria-expanded="false" and @class="inline-block cursor-pointer"]'
        '/span[@class="leading-tight text-lg tablet:text-2xl font-normal" and contains(@aria-label,"Show")]'
    ))).text, end = ' ')
if len(browser.find_elements(
        by = 'xpath',
        value = '//div[@class="pt-4"]/p[@class="font-medium text-light pt-2"]'
)) > 0:
    print('(Docuseries)')
else:
    print('(Documentary)')

可复现问题的测试代码 #

from contextlib import suppress
from os import getpid, kill
from re import compile
from signal import SIGTERM
from time import sleep  # noqa
import selenium.common.exceptions as exc
import selenium.webdriver.support.expected_conditions as ec
from selenium.webdriver import Firefox
from selenium.webdriver.support.wait import WebDriverWait
# from library import Firefox
if __name__ == '__main__':
    browser = Firefox()
    wait = WebDriverWait(driver = browser, timeout = 30)
    browser.set_window_rect(x = 960, y = 10, width = 1920, height = 1580)
    browser.get('https://curiositystream.com/')
    url = compile(r'https://curiositystream.com/video/[0-9]+')
    current_url, current_title = browser.current_url, browser.title
    try:
        while True:
            if current_url != browser.current_url:
                current_url = browser.current_url
                wait.until_not(ec.title_is(current_title))
                if url.match(string = browser.current_url):
                    current_title = browser.title
                    if (button := next((_ for _ in browser.find_elements(
                            by = 'xpath',
                            value = '//button[@class="vjs-big-play-button" and '
                                    '@type="button" and '
                                    '@title="Play Video" and '
                                    '@aria-disabled="false"]'
                    )), None)) is not None:
                        with suppress(
                                exc.StaleElementReferenceException,
                                exc.ElementNotInteractableException
                        ):
                            button.click()
                    print(wait.until(
                        ec.visibility_of_element_located((
                            'xpath',
                            '//button[@aria-expanded="false" and @class="inline-block cursor-pointer"]'
                            '/span[@class="leading-tight text-lg tablet:text-2xl font-normal" and contains(@aria-label,"Show")]'
                        ))).text, end = ' ')
                    if len(browser.find_elements(
                            by = 'xpath',
                            value = '//div[@class="pt-4"]/p[@class="font-medium text-light pt-2"]'
                    )):
                        print('(Docuseries)')
                    else:
                        print('(Documentary)')
    except exc.NoSuchWindowException:
        kill(getpid(), SIGTERM)

稳定解决方案 #

方法1：等待标题连续稳定 #

核心思路是等待标题连续多次保持一致，确认后台加载完成，避免捕获到中间状态：

import time
from selenium.common.exceptions import TimeoutException

def wait_for_stable_title(driver, initial_title, timeout=30, poll_frequency=0.5):
    end_time = time.time() + timeout
    last_title = initial_title
    # 连续3次获取到相同标题，确认加载完成
    stable_threshold = 3
    stable_count = 0
    
    while time.time() < end_time:
        current_title = driver.title
        if current_title == last_title:
            stable_count += 1
            if stable_count >= stable_threshold:
                return current_title
        else:
            stable_count = 0
            last_title = current_title
        time.sleep(poll_frequency)
    raise TimeoutException("标题未在超时时间内稳定")

# 测试代码中替换原有标题等待逻辑
if current_url != browser.current_url:
    current_url = browser.current_url
    # 先等待标题开始变化
    wait.until_not(ec.title_is(current_title))
    # 再等待标题稳定，获取最终系列标题
    current_title = wait_for_stable_title(browser, current_title)
    if url.match(current_url):
        # 后续获取元素、判断类型逻辑不变
        ...

方法2：等待目标元素文本稳定 #

针对显示系列标题的元素，等待其文本连续稳定，确保内容加载完成：

import time
from selenium.common.exceptions import TimeoutException

def wait_for_stable_element_text(driver, locator, timeout=30, poll_frequency=0.5):
    element = wait.until(ec.visibility_of_element_located(locator))
    end_time = time.time() + timeout
    last_text = element.text.strip()
    stable_threshold = 3
    stable_count = 0
    
    while time.time() < end_time:
        current_text = element.text.strip()
        if current_text == last_text:
            stable_count += 1
            if stable_count >= stable_threshold:
                return current_text
        else:
            stable_count = 0
            last_text = current_text
        time.sleep(poll_frequency)
    raise TimeoutException("元素文本未在超时时间内稳定")

# 使用示例替换原有元素获取逻辑
title_locator = (
    'xpath',
    '//button[@aria-expanded="false" and @class="inline-block cursor-pointer"]'
    '/span[@class="leading-tight text-lg tablet:text-2xl font-normal" and contains(@aria-label,"Show")]'
)
final_title = wait_for_stable_element_text(browser, title_locator)
print(final_title, end=' ')

方法3：监听后台API请求完成 #

通过监听Curiosity Stream加载系列信息的API请求，确保数据加载完成后再获取标题（仅Firefox可用）：

import json
import re
import time
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

# 初始化浏览器时配置网络日志
caps = DesiredCapabilities.FIREFOX
caps['loggingPrefs'] = {'performance': 'ALL'}
options = Options()
options.set_preference('devtools.netmonitor.enabled', True)
browser = Firefox(options=options, desired_capabilities=caps)

def wait_for_api_request(driver, url_pattern, timeout=30):
    end_time = time.time() + timeout
    while time.time() < end_time:
        logs = driver.get_log('performance')
        for log in logs:
            try:
                log_json = json.loads(log['message'])
                msg = log_json['message']
                if msg['method'] == 'Network.responseReceived':
                    request_url = msg['params']['request']['url']
                    if re.search(url_pattern, request_url):
                        return True
            except (KeyError, json.JSONDecodeError):
                continue
        time.sleep(0.5)
    raise TimeoutException("未捕获到目标API请求")

# 使用示例：假设系列信息API包含'/series'关键字
wait_for_api_request(browser, r'/series')
# 此时获取标题或元素文本即为最终状态
final_title = browser.title

内容的提问来源于stack exchange，提问作者phpjunkie