使用requests爬取Wayfair德站沙发名称遇400错误，求解决方法

我尝试用requests模块爬取Wayfair德国站沙发分类页面的沙发名称，编写了如下代码。观察网页的网络请求逻辑后，确认代码逻辑和网页请求一致，但始终返回400状态码。请问如何用requests模块成功爬取该页面的沙发名称？

import json
import requests

url = "https://www.wayfair.de/graphql?hash=22125c9a747b989fb251b014c2628213%23adda13419b757ac36f2b6c49a3bcd81c%23d0e1538c8199ca3253f0ba6c6f96b840%23f905cea77203534e928de4366ee3779d%2356bc66914e917daca7b7b6fbf0a6bcbb%2385a60692d411b1b94fcaf7e769b299f7%23937b5986d523ec2cd8ea2e68291b69e4%23226dd1f891ead00134d0c77687add073%2369fcc9e7651b988309f3197763ba4579%23d878a5fca245086d6cd18dec8a364f30%2383d1f1132338d12cf533926fa527c5a7%233d54fb8f7ef35c34629aa853236e0fb2%23201c0252e9b59dc9bb102e35e1c303fe%2382032c3ee9d609c60f37d9172e2dfebe%2370ac03a548872795355b4b2d9ee86698%23c5a76122f3b3d9dddfaccf682f3c5e19%23adc79abec9f62457c811fbde536bcc59%23e5ba723f1dbb69724c115115fd923a47%23787e38482534a0d1832e19a21a36700e%239c61a681637cd2ae4bc71227fe3cf711%2348c76b09e216b294dd5eaba432f05598%23ddeed8a890c5e7e562f69812d05c1cc7"

payload = {"variables":{"categoryId":479496,"browseInput":{"sortId":189,"filters":[],"pagination":{"page":2,"itemsPerPage":48},"boostedSkus":[],"isAjax":True,"skipLoadPricingModel":False},"usePricingField":True},"_pageId":"i1g5H18AMh07TKM2idhtCw==","_isPageRequest":True}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
    'Accept-Encoding': 'gzip, deflate, br, zstd',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept': 'application/json',
    'Origin': 'https://www.wayfair.de',
    'Referer': 'https://www.wayfair.de/moebel/sb0/sofas-c479496.html',
    'X-Parent-Txid': 'I/bHHmXsMEuIX3rEX/ejAg==',
    'Apollographql-Client-Name': '@wayfair/sf-ui-browse',
    'Apollographql-Client-Version': '6c7f9310e161d153c0d6b90b5f2961dae465981e',
}

with requests.Session() as s:
    s.headers.update(headers)
    res = s.post(url, json=payload)
    print(res.status_code)
    print(res.json())

问题原因 #

返回400状态码的核心原因是硬编码了多个动态生成的参数，这些参数每次会话都会变化，服务器会验证其有效性：

• X-Parent-Txid：会话级事务ID，每次访问都会生成新值
• _pageId：页面唯一标识，随页面加载动态生成
• URL中的hash：GraphQL查询的签名哈希，可能随页面版本或查询内容变化
• 缺失operationName：GraphQL请求的必填参数，原始payload未包含

解决方案 #

先通过GET请求访问目标页面，提取必要的动态参数，再发送GraphQL POST请求：

import json
import requests
from bs4 import BeautifulSoup
import re
import base64
import os

# 目标页面URL
base_url = "https://www.wayfair.de/moebel/sb0/sofas-c479496.html"
graphql_url = "https://www.wayfair.de/graphql"

with requests.Session() as s:
    # 1. 先GET页面，获取动态参数和会话cookie
    s.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
        'Accept-Language': 'en-US,en;q=0.9'
    })
    response = s.get(base_url)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 提取_pageId：从页面脚本中匹配
    page_id = ""
    for script in soup.find_all('script'):
        if '_pageId' in script.text:
            match = re.search(r'"_pageId":"([^"]+)"', script.text)
            if match:
                page_id = match.group(1)
                break
    
    # 生成X-Parent-Txid：模拟随机base64格式的事务ID
    x_parent_txid = base64.b64encode(os.urandom(16)).decode('utf-8')
    
    # 2. 构造请求头
    headers = {
        'Accept': 'application/json',
        'Origin': 'https://www.wayfair.de',
        'Referer': base_url,
        'X-Parent-Txid': x_parent_txid,
        'Apollographql-Client-Name': '@wayfair/sf-ui-browse',
        'Apollographql-Client-Version': '6c7f9310e161d153c0d6b90b5f2961dae465981e',
    }
    
    # 3. 构造完整payload，添加缺失的operationName
    payload = {
        "operationName": "BrowseCategoryQuery",
        "variables": {
            "categoryId": 479496,
            "browseInput": {
                "sortId": 189,
                "filters": [],
                "pagination": {"page": 2, "itemsPerPage": 48},
                "boostedSkus": [],
                "isAjax": True,
                "skipLoadPricingModel": False
            },
            "usePricingField": True
        },
        "_pageId": page_id,
        "_isPageRequest": True
    }
    
    # 4. 发送POST请求
    res = s.post(graphql_url, json=payload, headers=headers)
    
    # 处理响应并提取沙发名称
    if res.status_code == 200:
        data = res.json()
        products = data['data']['browse']['products']['items']
        for idx, product in enumerate(products, 1):
            print(f"{idx}. {product['name']}")
    else:
        print(f"请求失败，状态码：{res.status_code}")
        print(res.text)

关键注意事项 #

• 动态参数实时获取：_pageId、X-Parent-Txid必须从当前会话的页面中提取，不能硬编码
• 必填参数补全：GraphQL请求必须包含operationName字段
• 会话一致性：使用同一个Session对象处理所有请求，确保cookie等会话信息一致
• 反爬规避：Wayfair有反爬机制，建议添加请求间隔，避免短时间内大量请求

内容的提问来源于stack exchange，提问作者MITHU