无需硬编码Cookie，如何用Requests爬取Zillow的Zestimate？

我尝试使用requests模块从Zillow房价估值页面爬取地址1205 Clover St, Accokeek, MD 20607的Zestimate值。流程是在网页输入地址后得到zpid37374749，将其放入POST请求参数即可获取数据，但当前脚本仅在硬编码从开发者工具获取的Cookie时才能正常运行，否则会抛出JSONDecodeError。现有代码如下：

import requests
from pprint import pprint
from bs4 import BeautifulSoup

url = 'https://www.zillow.com/graphql/'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
    'referer': 'https://www.zillow.com/how-much-is-my-home-worth/',
    'cookie': '_px3=00d7bf4fc2e331a793b7ebf9f9e8059013b86b39ead67dfc15fd89fe4e1b459c:GrviXjUIrsA7lYSeSBB8Y8yQedr9ng0ByK7xkaB6CLDFCGYjxBL2r8dPy6MwnUk0Eqv75g0t1fcIa4c8ksIHoA==:1000:1qeUILwk/42tR6FaZ0ybaI63rAWDBAc7fsAeLcH02MyzXKEEoT0SDnk/zMCLA3Zj1BpBmuqO/4RQDntxiOgtsCGPF1VuxfpmXAeMPcxegynJ/PGXWYE3OLIa9vJ9XkVkGaSoH9knD3Ls7nHyuP/0DhapZbThlzDLoQ//Vflzi0eGv12lCSX/2msW+rCeFgO0RhsBMNSN93EUhmuMBXcBMg==;'

}
params = {"operationName":"HowMuchIsMyHomeWorthReviewQuery","variables":{"zpid":37374749},"query":"query HowMuchIsMyHomeWorthReviewQuery($zpid: ID!) {\n  property(zpid: $zpid) {\n    streetAddress\n    city\n    state\n    zipcode\n    bedrooms\n    bathrooms\n    livingArea\n    zestimate\n    homeStatus\n    photos(size: XL) {\n      url\n      __typename\n    }\n    ...OmpHomeWorthUpsell_property\n    isConfirmedClaimedByCurrentSignedInUser\n    isVerifiedClaimedByCurrentSignedInUser\n    ...UARequiredPropertyDimensions_property\n    ...ContactAgentForm_property\n    ...HomeInfo_property\n    __typename\n  }\n  viewer {\n    ...ContactAgentForm_viewer\n    __typename\n  }\n  abTests {\n    ...OmpHomeWorthUpsell_abTests\n    ...UARequiredPropertyDimensions_abTests\n    ...ContactAgentForm_abTests\n    __typename\n  }\n}\n\nfragment OmpHomeWorthUpsell_property on Property {\n  zpid\n  onsiteMessage(placementNames: [\"HMIMHWTopSlot\"]) {\n    ...onsiteMessage_fragment\n    __typename\n  }\n  __typename\n}\n\nfragment onsiteMessage_fragment on OnsiteMessageResultType {\n  eventId\n  decisionContext\n  messages {\n    skipDisplayReason\n    shouldDisplay\n    isGlobalHoldout\n    isPlacementHoldout\n    placementName\n    testPhase\n    bucket\n    placementId\n    passThrottle\n    lastModified\n    eventId\n    decisionContext\n    selectedTreatment {\n      id\n      name\n      component\n      status\n      renderingProps\n      lastModified\n      __typename\n    }\n    qualifiedTreatments {\n      id\n      name\n      status\n      lastModified\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n\nfragment OmpHomeWorthUpsell_abTests on ABTests {\n  HMIMHW_ZO_NFS_UPSELL_ONSITE_MESSAGING: abTest(\n    trial: \"HMIMHW_ZO_NFS_UPSELL_ONSITE_MESSAGING\"\n  )\n  __typename\n}\n\nfragment UARequiredPropertyDimensions_property on Property {\n  currency\n  featuredListingTypeDimension\n  hasPublicVideo\n  hdpTypeDimension\n  listingTypeDimension\n  price\n  propertyTypeDimension\n  standingOffer {\n    isStandingOfferEligible\n    __typename\n  }\n  zpid\n  isZillowOwned\n  zillowOfferMarket {\n    legacyName\n    __typename\n  }\n  ...ShouldShowVideo_property\n  __typename\n}\n\nfragment ShouldShowVideo_property on Property {\n  homeStatus\n  isZillowOwned\n  hasPublicVideo\n  primaryPublicVideo {\n    sources {\n      src\n      __typename\n    }\n    __typename\n  }\n  richMediaVideos {\n    mp4Url\n    hlsUrl\n    __typename\n  }\n  __typename\n}\n\nfragment UARequiredPropertyDimensions_abTests on ABTests {\n  ZO_HDP_HOUR_ONE_VIDEO: abTest(trial: \"ZO_HDP_HOUR_ONE_VIDEO\")\n  __typename\n}\n\nfragment ContactAgentForm_property on Property {\n  streetAddress\n  state\n  city\n  zipcode\n  zpid\n  homeStatus\n  homeType\n  zestimate\n  homeType\n  isInstantOfferEnabled\n  zillowOfferMarket {\n    name\n    code\n    __typename\n  }\n  __typename\n}\n\nfragment ContactAgentForm_viewer on Viewer {\n  name\n  email\n  zuid\n  __typename\n}\n\nfragment ContactAgentForm_abTests on ABTests {\n  SHOW_PL_LEAD_FORM: abTest(trial: \"SHOW_PL_LEAD_FORM\")\n  __typename\n}\n\nfragment HomeInfo_property on Property {\n  streetAddress\n  city\n  state\n  zipcode\n  bedrooms\n  bathrooms\n  livingArea\n  homeStatus\n  homeType\n  contingentListingType\n  photos(size: XL) {\n    url\n    __typename\n  }\n  listing_sub_type {\n    is_newHome\n    is_FSBO\n    is_bankOwned\n    is_foreclosure\n    is_forAuction\n    is_comingSoon\n    __typename\n  }\n  __typename\n}\n"}

with requests.Session() as s:
    s.headers.update(headers)
    res = s.post(url,json=params)
    pprint(res.json()['data']['property']['zestimate'])

Zillow的GraphQL接口依赖动态生成的反爬Cookie（如_px3字段，来自PerimeterX反爬系统），硬编码Cookie无法长期复用。以下是两种无需硬编码Cookie的实现方案：

方案一：用requests.Session自动获取会话Cookie #

先访问初始页面让服务器生成会话Cookie，再发送GraphQL请求，利用Session自动维护Cookie：

import requests
from pprint import pprint

# 初始页面URL
initial_url = "https://www.zillow.com/how-much-is-my-home-worth/"
graphql_url = "https://www.zillow.com/graphql/"

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'referer': 'https://www.zillow.com/how-much-is-my-home-worth/',
}

params = {"operationName":"HowMuchIsMyHomeWorthReviewQuery","variables":{"zpid":37374749},"query":"query HowMuchIsMyHomeWorthReviewQuery($zpid: ID!) {\n  property(zpid: $zpid) {\n    streetAddress\n    city\n    state\n    zipcode\n    bedrooms\n    bathrooms\n    livingArea\n    zestimate\n    homeStatus\n    photos(size: XL) {\n      url\n      __typename\n    }\n    ...OmpHomeWorthUpsell_property\n    isConfirmedClaimedByCurrentSignedInUser\n    isVerifiedClaimedByCurrentSignedInUser\n    ...UARequiredPropertyDimensions_property\n    ...ContactAgentForm_property\n    ...HomeInfo_property\n    __typename\n  }\n  viewer {\n    ...ContactAgentForm_viewer\n    __typename\n  }\n  abTests {\n    ...OmpHomeWorthUpsell_abTests\n    ...UARequiredPropertyDimensions_abTests\n    ...ContactAgentForm_abTests\n    __typename\n  }\n}\n\nfragment OmpHomeWorthUpsell_property on Property {\n  zpid\n  onsiteMessage(placementNames: [\"HMIMHWTopSlot\"]) {\n    ...onsiteMessage_fragment\n    __typename\n  }\n  __typename\n}\n\nfragment onsiteMessage_fragment on OnsiteMessageResultType {\n  eventId\n  decisionContext\n  messages {\n    skipDisplayReason\n    shouldDisplay\n    isGlobalHoldout\n    isPlacementHoldout\n    placementName\n    testPhase\n    bucket\n    placementId\n    passThrottle\n    lastModified\n    eventId\n    decisionContext\n    selectedTreatment {\n      id\n      name\n      component\n      status\n      renderingProps\n      lastModified\n      __typename\n    }\n    qualifiedTreatments {\n      id\n      name\n      status\n      lastModified\n      __typename\n    }\n    __typename\n  }\n  __typename\n}\n\nfragment OmpHomeWorthUpsell_abTests on ABTests {\n  HMIMHW_ZO_NFS_UPSELL_ONSITE_MESSAGING: abTest(\n    trial: \"HMIMHW_ZO_NFS_UPSELL_ONSITE_MESSAGING\"\n  )\n  __typename\n}\n\nfragment UARequiredPropertyDimensions_property on Property {\n  currency\n  featuredListingTypeDimension\n  hasPublicVideo\n  hdpTypeDimension\n  listingTypeDimension\n  price\n  propertyTypeDimension\n  standingOffer {\n    isStandingOfferEligible\n    __typename\n  }\n  zpid\n  isZillowOwned\n  zillowOfferMarket {\n    legacyName\n    __typename\n  }\n  ...ShouldShowVideo_property\n  __typename\n}\n\nfragment ShouldShowVideo_property on Property {\n  homeStatus\n  isZillowOwned\n  hasPublicVideo\n  primaryPublicVideo {\n    sources {\n      src\n      __typename\n    }\n    __typename\n  }\n  richMediaVideos {\n    mp4Url\n    hlsUrl\n    __typename\n  }\n  __typename\n}\n\nfragment UARequiredPropertyDimensions_abTests on ABTests {\n  ZO_HDP_HOUR_ONE_VIDEO: abTest(trial: \"ZO_HDP_HOUR_ONE_VIDEO\")\n  __typename\n}\n\nfragment ContactAgentForm_property on Property {\n  streetAddress\n  state\n  city\n  zipcode\n  zpid\n  homeStatus\n  homeType\n  zestimate\n  homeType\n  isInstantOfferEnabled\n  zillowOfferMarket {\n    name\n    code\n    __typename\n  }\n  __typename\n}\n\nfragment ContactAgentForm_viewer on Viewer {\n  name\n  email\n  zuid\n  __typename\n}\n\nfragment ContactAgentForm_abTests on ABTests {\n  SHOW_PL_LEAD_FORM: abTest(trial: \"SHOW_PL_LEAD_FORM\")\n  __typename\n}\n\nfragment HomeInfo_property on Property {\n  streetAddress\n  city\n  state\n  zipcode\n  bedrooms\n  bathrooms\n  livingArea\n  homeStatus\n  homeType\n  contingentListingType\n  photos(size: XL) {\n    url\n    __typename\n  }\n  listing_sub_type {\n    is_newHome\n    is_FSBO\n    is_bankOwned\n    is_foreclosure\n    is_forAuction\n    is_comingSoon\n    __typename\n  }\n  __typename\n}\n"}

with requests.Session() as s:
    # 先访问初始页面，获取服务器生成的会话Cookie
    s.get(initial_url, headers=headers)
    # 发送GraphQL请求
    res = s.post(graphql_url, json=params, headers=headers)
    try:
        data = res.json()
        pprint(data['data']['property']['zestimate'])
    except requests.exceptions.JSONDecodeError:
        print("请求被拦截，需处理反爬验证")
        print("响应内容：", res.text)

方案二：用Selenium模拟真实浏览器（更稳定） #

如果纯requests方式被拦截，用Selenium模拟用户操作，自动处理Cookie和反爬验证：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

# 初始化Chrome浏览器
driver = webdriver.Chrome()
driver.get("https://www.zillow.com/how-much-is-my-home-worth/")

# 等待地址输入框加载并输入目标地址
wait = WebDriverWait(driver, 10)
address_input = wait.until(EC.presence_of_element_located((By.ID, "search-box-input")))
address_input.send_keys("1205 Clover St, Accokeek, MD 20607")

# 点击搜索按钮
search_btn = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[data-testid='search-button']")))
search_btn.click()

# 等待页面加载完成，提取Zestimate值
time.sleep(3)
zestimate = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "[data-testid='zestimate-value']"))).text
print("Zestimate值：", zestimate)

driver.quit()

关键说明 #

• Zillow反爬机制严格，_px3 Cookie是动态生成的，每次会话都会变化，硬编码无法长期使用。
• 方案一依赖Session自动维护Cookie，但如果触发验证码仍会失败，适合简单场景。
• 方案二模拟真实用户行为，能自动处理大多数反爬验证，稳定性更高。

内容的提问来源于stack exchange，提问作者SMTH