使用Selenium Python获取iframe中HTTPS请求的问题排查

核心问题根源 #

• 日志捕获上下文错误：打开新标签页后未切换到对应窗口句柄，导致获取的是原窗口的网络日志
• 请求丢失：触发POST请求后页面自动跳转到Gmail，中断了日志捕获流程

步骤1：切换到新标签页的窗口上下文 #

新标签页属于独立的浏览器窗口，必须切换到其句柄才能捕获该窗口的网络事件：

# 点击打开新标签页后，执行以下代码切换上下文
original_window = driver.current_window_handle
# 等待新标签页加载完成
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
# 遍历窗口句柄，切换到新窗口
for window_handle in driver.window_handles:
    if window_handle != original_window:
        driver.switch_to.window(window_handle)
        break

步骤2：用Chrome DevTools Protocol(CDP)实时监听+拦截跳转 #

传统的get_log是一次性拉取日志，无法捕获pending状态的请求，且跳转后日志会丢失。改用CDP实现实时监听，并拦截跳转到Gmail的请求：

初始化CDP监听与拦截 #

# 启用网络监听
driver.execute_cdp_cmd('Network.enable', {})

# 存储捕获到的POST请求
captured_post_requests = []

# 监听请求发送事件，筛选HTTPS POST请求
def capture_post_request(event):
    req = event['params']['request']
    if req['method'] == 'POST' and req['url'].startswith('https'):
        captured_post_requests.append(req)

# 绑定监听事件
driver.add_cdp_listener('Network.requestWillBeSent', capture_post_request)

# 拦截跳转到Gmail的请求，避免页面跳转丢失日志
def block_gmail_redirect(event):
    interception_id = event['params']['interceptionId']
    req_url = event['params']['request']['url']
    if 'gmail.com' in req_url:
        # 取消跳转请求
        driver.execute_cdp_cmd('Network.continueInterceptedRequest', {
            'interceptionId': interception_id,
            'cancel': True
        })
    else:
        # 放行其他正常请求
        driver.execute_cdp_cmd('Network.continueInterceptedRequest', {
            'interceptionId': interception_id
        })

# 启用请求拦截规则
driver.execute_cdp_cmd('Network.setRequestInterception', {
    'patterns': [{'urlPattern': '*', 'resourceType': 'Document'}]
})
driver.add_cdp_listener('Network.requestIntercepted', block_gmail_redirect)

完整修改后的代码 #

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
import json
import datetime
import os

# 配置Chrome Driver
chrome_options = Options()
chrome_options.add_experimental_option("detach", True)
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument('--auto-open-devtools-for-tabs')
chrome_options.add_argument('--log-level=2')
chrome_options.add_argument('--disable-features=IsolateOrigins,site-per-process')
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])

capabilities = webdriver.DesiredCapabilities.CHROME
capabilities["goog:loggingPrefs"] = {"performance": "ALL",'browser':'ALL','server':'ALL'}
capabilities['acceptSslCerts'] = True
capabilities['acceptInsecureCerts'] = True
capabilities['PageLoadStrategy'] = 'none'

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options,desired_capabilities=capabilities)
driver.get(os.environ['URL'])

# 执行登录流程（请补充你的登录代码）
# ...

# 点击按钮打开新标签页（请补充打开新标签的触发代码）
# driver.find_element(By.XPATH, "//按钮定位").click()

# 切换到新标签页
original_window = driver.current_window_handle
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
for window_handle in driver.window_handles:
    if window_handle != original_window:
        driver.switch_to.window(window_handle)
        break

# 初始化CDP网络监听与拦截
driver.execute_cdp_cmd('Network.enable', {})
captured_post_requests = []

def capture_post_request(event):
    req = event['params']['request']
    if req['method'] == 'POST' and req['url'].startswith('https'):
        captured_post_requests.append(req)

driver.add_cdp_listener('Network.requestWillBeSent', capture_post_request)

def block_gmail_redirect(event):
    interception_id = event['params']['interceptionId']
    req_url = event['params']['request']['url']
    if 'gmail.com' in req_url:
        driver.execute_cdp_cmd('Network.continueInterceptedRequest', {
            'interceptionId': interception_id,
            'cancel': True
        })
    else:
        driver.execute_cdp_cmd('Network.continueInterceptedRequest', {
            'interceptionId': interception_id
        })

driver.execute_cdp_cmd('Network.setRequestInterception', {
    'patterns': [{'urlPattern': '*', 'resourceType': 'Document'}]
})
driver.add_cdp_listener('Network.requestIntercepted', block_gmail_redirect)

# 切换到iframe并触发请求
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[@id="myiframe"]')))
row_selector = '//*[@id="root"]/div/div/div/div[2]/div[2]/div/div/div/div/table/tbody/tr[1]/th[3]'
row_selector_clickable = '//*[@id="root"]/div/div/div/div[2]/div[2]/div/div/div/div/table/tbody/tr[1]/th[3]/div/div/div[2]/div/button'

WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.XPATH, row_selector)))
actions = ActionChains(driver)
row_element = driver.find_element(By.XPATH, row_selector)
row_clickable = driver.find_element(By.XPATH, row_selector_clickable)
actions.move_to_element(row_element)
actions.click(row_clickable)
actions.perform()

# 等待请求完成（根据实际请求耗时调整等待时间）
WebDriverWait(driver, 15).until(lambda d: len(captured_post_requests) > 0)

# 停止监听并保存日志
driver.execute_cdp_cmd('Network.disable', {})
with open(f"log_entries-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}.txt", "wt") as out:
    out.write(json.dumps(captured_post_requests, indent=2))

# 后续清理操作
# driver.quit()

关键说明 #

1. 窗口句柄切换：新标签页是独立的浏览器上下文，必须切换后才能捕获其网络日志
2. CDP实时监听：相比get_log，CDP可以实时捕获请求全生命周期，不会因页面跳转丢失日志
3. 跳转拦截：阻止跳转到Gmail，确保请求完成后再处理日志
4. 等待请求完成：通过WebDriverWait等待POST请求被捕获，避免过早保存空日志

内容的提问来源于stack exchange，提问作者Manspof