如何用Python+Selenium抓取浏览器Network标签中restaurant文件的响应内容

核心原因说明 #

你当前只能拿到请求发起事件的原因是：Network.requestWillBeSent 仅记录请求发出的元数据，响应内容不会直接存在性能日志中，需要调用Chrome DevTools Protocol（CDP）的专用接口主动获取。

完整可运行代码 #

import time
import json
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import DesiredCapabilities

def get_actions(driver):
    return ActionChains(driver)

def setup():
    # 配置性能日志采集
    desired_capabilities = DesiredCapabilities.CHROME
    desired_capabilities["goog:loggingPrefs"] = {"performance": "ALL"}
    # 注意替换为你本地的chromedriver路径
    driver = webdriver.Chrome(
        desired_capabilities=desired_capabilities, 
        executable_path=r"C:\\Users\\visha\\PycharmProjects\\grabScrapper\\chromedriver.exe"
    )

    address = 'Manila '
    driver.get('https://food.grab.com/ph/en/restaurants')
    time.sleep(2)
    driver.find_element_by_class_name("textPlaceholder___1yEAK").click()

    actions = get_actions(driver)
    actions.send_keys(address)
    actions.perform()
    time.sleep(1)

    actions = get_actions(driver)
    actions.send_keys(Keys.ARROW_DOWN)
    actions.perform()
    time.sleep(1)

    actions = get_actions(driver)
    actions.send_keys(Keys.ENTER)
    actions.perform()

    time.sleep(5)
    driver.refresh()
    time.sleep(8) # 延长等待时间确保页面完全加载
    
    # 处理性能日志获取响应
    logs = driver.get_log("performance")
    target_request_id = None
    # 筛选目标文档的响应接收事件
    for entry in logs:
        log = json.loads(entry["message"])["message"]
        if log["method"] == "Network.responseReceived":
            params = log["params"]
            if params["type"] == "Document" and params["response"]["url"] == "https://food.grab.com/ph/en/restaurants":
                target_request_id = params["requestId"]
                break
    
    if target_request_id:
        # 调用CDP接口拉取响应体
        resp = driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": target_request_id})
        # resp["body"]就是完整的页面响应HTML
        print("响应内容前1000字符预览：")
        print(resp["body"][:1000])
        # 此处添加你的自定义处理逻辑
    else:
        print("未匹配到目标页面的响应记录")
    
    return driver

if __name__ == '__main__':
    driver = setup()
    # 处理完成后可手动关闭driver
    # driver.quit()

适配Selenium 4.x版本的配置调整 #

如果你使用的是Selenium 4.x版本，原有的DesiredCapabilities配置可以替换为更规范的Options写法：

from selenium.webdriver.chrome.options import Options
options = Options()
options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
driver = webdriver.Chrome(options=options, executable_path="你的chromedriver路径")

注意事项 #

• 请确保Chrome浏览器版本和chromedriver版本完全一致，否则CDP接口调用会出现异常
• 如果需要抓取页面后续滚动加载的接口数据，可以用同样逻辑筛选type为XHR/Fetch的请求，匹配对应接口URL后拉取响应体

内容的提问来源于stack exchange，提问作者Vishal Anand