使用selenium-wire筛选Network面板proxy-vip请求失败如何解决

使用selenium-wire捕获lovepik网站点击下载按钮触发的网络请求，需要提取形如https://proxy-vip.lovepik.com/...的目标下载地址，现有代码运行时始终无法匹配到目标请求，控制台持续打印"Not found"。
原实现代码如下：

import os
from seleniumwire import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
import time


def main():
    username = "adminbkh@gmail.com"
    password = "Adminbkh123@"
    chrome_options = Options()
    chrome_options.add_experimental_option(
                "excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option(
                'useAutomationExtension', False)
    chrome_options.add_experimental_option("prefs", {
                "download.prompt_for_download": False,
                "safebrowsing.enabled": True
            })
    driver = webdriver.Chrome(
                ChromeDriverManager().install(), options=chrome_options)

    driver.maximize_window()

    driver.get("https://lovepik.com/video-42018/ocean-waves-roll-and-hit-the-scene.html")
    driver.find_element(By.XPATH, '//div[@class="user"]//*[@onclick="env.login();"]').click()
    driver.find_element(
                By.XPATH, '//form[@id="form-login-v4"]//*[@name="email"]').send_keys(username)
    driver.find_element(
                By.XPATH, '//form[@id="form-login-v4"]//*[@name="password"]').send_keys(password)
    driver.find_element(By.XPATH, '//button[text()="Login "]').click()
    time.sleep(2)
    driver.switch_to.window(driver.window_handles[0])
    try:
        video = driver.find_element(By.XPATH, '//div[@class="limit_down"]//*[@class="down_t"]')
        if video.is_displayed():
            video.click()
            print("This is a file video")
            time.sleep(5)
            for request in driver.requests:
                if request.response:
                    if "proxy-vip" in request.url:
                        print("Found")
                    else:
                        print("Not found")
        driver.close()
    except NoSuchElementException:
        print("Couldn't find any elements")

if __name__ == "__main__":
    main()

• 遍历逻辑错误：每遍历到一个不匹配的请求就打印一次"Not found"，大量页面加载、登录相关的无关请求会输出海量"Not found"日志，极易覆盖掉可能存在的匹配成功日志，造成"始终找不到"的错觉
• 等待逻辑不可靠：固定sleep(5)的等待时长完全凭经验设置，点击下载按钮后目标请求可能还未发起、还未收到响应就开始遍历请求列表，自然无法捕获到目标
• 冗余请求干扰：从打开页面、登录到点击下载的全流程所有请求都存在于请求列表中，无关请求占比极高，提升了匹配漏判概率
• 新窗口请求漏抓：点击下载按钮大概率会触发新标签页弹出，代码未处理新窗口切换逻辑，新标签页上下文里的网络请求不会被当前窗口的监听逻辑捕获

核心调整点：

• 点击下载按钮前清空历史冗余请求，仅监听点击操作之后产生的新请求
• 用selenium-wire自带的请求等待API替代固定sleep，最长等待15秒直到目标请求出现，兼顾稳定性和执行效率
• 调整日志输出逻辑，遍历完全部请求后再统一输出匹配结果，避免无效日志刷屏
• 点击下载后自动检测新弹出的标签页，切换到新标签页监听请求

import os
from seleniumwire import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
import time


def main():
    username = "adminbkh@gmail.com"
    password = "Adminbkh123@"
    chrome_options = Options()
    chrome_options.add_experimental_option(
                "excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option(
                'useAutomationExtension', False)
    chrome_options.add_experimental_option("prefs", {
                "download.prompt_for_download": False,
                "safebrowsing.enabled": True
            })
    driver = webdriver.Chrome(
                ChromeDriverManager().install(), options=chrome_options)

    driver.maximize_window()

    driver.get("https://lovepik.com/video-42018/ocean-waves-roll-and-hit-the-scene.html")
    driver.find_element(By.XPATH, '//div[@class="user"]//*[@onclick="env.login();"]').click()
    driver.find_element(
                By.XPATH, '//form[@id="form-login-v4"]//*[@name="email"]').send_keys(username)
    driver.find_element(
                By.XPATH, '//form[@id="form-login-v4"]//*[@name="password"]').send_keys(password)
    driver.find_element(By.XPATH, '//button[text()="Login "]').click()
    time.sleep(2)
    # 切回原窗口，关闭登录可能弹出的其他窗口
    for handle in driver.window_handles:
        if handle != driver.current_window_handle:
            driver.switch_to.window(handle)
            driver.close()
    driver.switch_to.window(driver.window_handles[0])
    
    try:
        video = driver.find_element(By.XPATH, '//div[@class="limit_down"]//*[@class="down_t"]')
        if video.is_displayed():
            # 点击下载前清空所有历史请求，避免无关请求干扰
            driver.requests.clear()
            origin_window_count = len(driver.window_handles)
            video.click()
            print("触发下载操作，等待目标请求...")
            
            # 检测是否有新窗口弹出，有则切换
            time.sleep(1)
            if len(driver.window_handles) > origin_window_count:
                driver.switch_to.window(driver.window_handles[-1])
            
            target_url = None
            try:
                # 等待匹配规则的请求出现，最长等待15秒
                target_request = driver.wait_for_request(r'.*proxy-vip\.lovepik\.com.*', timeout=15)
                target_url = target_request.url
                print(f"找到目标下载地址：{target_url}")
            except TimeoutException:
                # 等待超时后手动遍历所有已捕获的请求二次确认
                for request in driver.requests:
                    if request.response and "proxy-vip.lovepik.com" in request.url:
                        target_url = request.url
                        print(f"找到目标下载地址：{target_url}")
                        break
            
            if not target_url:
                print("未找到匹配的proxy-vip请求")

        driver.quit()
    except NoSuchElementException:
        print("页面元素定位失败")

if __name__ == "__main__":
    main()

注意：如果运行后还是抓不到请求，可以检查selenium-wire版本，部分旧版本对HTTPS/HTTP2的请求捕获需要额外安装根证书，按照启动时的控制台提示安装证书后即可正常捕获HTTPS请求。

内容的提问来源于stack exchange，提问作者Blog Khóa Học