使用requests模块爬取realtor.com代理数据遇授权错误

访问realtor.com的经纪人页面，在「City or zip」输入框填入Miami, FL并点击搜索可正常显示结果，但运行以下requests脚本时，返回错误：

You are not authorized to access this request.

用户使用的代码：

import json
import requests
from pprint import pprint
from bs4 import BeautifulSoup

URL = "https://www.realtor.com/realestateagents/"
link = 'https://www.realtor.com/realestateagents/api/v3/search'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'referer': 'https://www.realtor.com/realestateagents/',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'en-US,en;q=0.9,bn;q=0.8',
    'X-Requested-With': 'XMLHttpRequest',
    'x-newrelic-id': 'VwEPVF5XGwQHXFNTBAcAUQ==',
    'authorization': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NjQ1MjU0NDQsInN1YiI6ImZpbmRfYV9yZWFsdG9yIiwiaWF0IjoxNjY0NTI0Nzk2fQ.Q2jryTAD5vgsJ37e1SylBnkaeK7Cln930Q8KL4ANqsM'
}

params = {
    'nar_only': '1',
    'offset': '',
    'limit': '20',
    'marketing_area_cities': 'FL_Miami',
    'postal_code': '',
    'is_postal_search': 'true',
    'name': '',
    'types': 'agent',
    'sort': 'recent_activity_high',
    'far_opt_out': 'false',
    'client_id': 'FAR2.0',
    'recommendations_count_min': '',
    'agent_rating_min': '',
    'languages': '',
    'agent_type': '',
    'price_min': '',
    'price_max': '',
    'designations': '',
    'photo': 'true',
    'seoUserType': "{'isBot':'false','deviceType':'desktop'}",
    'is_county_search': 'false',
    'county': ''
}

with requests.Session() as s:
    s.headers.update(headers)
    res = s.get(link,params=params)
    print(res.status_code)
    print(res.json())

核心问题 #

硬编码的authorization和x-newrelic-id是临时会话令牌，会过期且与用户会话绑定，直接复用会导致授权失败。此外部分参数格式错误也会影响请求合法性。

修改后的代码 #

import requests
from bs4 import BeautifulSoup
import json

BASE_URL = "https://www.realtor.com/realestateagents/"
SEARCH_API = "https://www.realtor.com/realestateagents/api/v3/search"

# 初始化会话，设置基础请求头
session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
    'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8',
    'Referer': 'https://www.realtor.com/'
})

# 先访问主页面，建立有效会话并获取动态令牌
response = session.get(BASE_URL)
soup = BeautifulSoup(response.text, 'html.parser')

# 从页面初始化数据中提取authorization令牌
initial_state_script = soup.find('script', id='__NEXT_DATA__')
if initial_state_script:
    initial_state = json.loads(initial_state_script.string)
    auth_token = initial_state.get('props', {}).get('pageProps', {}).get('auth', {}).get('token')
    if auth_token:
        session.headers.update({'authorization': f'Bearer {auth_token}'})

# 从响应头提取x-newrelic-id
newrelic_id = response.headers.get('X-NewRelic-ID')
if newrelic_id:
    session.headers.update({'x-newrelic-id': newrelic_id})

# 修正搜索参数
params = {
    'nar_only': '1',
    'offset': '0',
    'limit': '20',
    'marketing_area_cities': 'FL_Miami',
    'postal_code': '',
    'is_postal_search': 'false',  # 城市搜索而非邮编搜索，设为false
    'name': '',
    'types': 'agent',
    'sort': 'recent_activity_high',
    'far_opt_out': 'false',
    'client_id': 'FAR2.0',
    'photo': 'true',
    'seoUserType': '{"isBot":"false","deviceType":"desktop"}',  # 用双引号符合JSON格式
    'is_county_search': 'false'
}

# 发起搜索请求
search_response = session.get(SEARCH_API, params=params)
print(f"状态码: {search_response.status_code}")
if search_response.status_code == 200:
    print(json.dumps(search_response.json(), indent=2))
else:
    print(f"错误响应: {search_response.text}")

关键调整说明 #

1. 动态获取令牌：先访问主页面，从页面初始化脚本或响应头中提取authorization和x-newrelic-id，避免硬编码过期令牌。
2. 修正参数格式：seoUserType必须是合法JSON字符串，使用双引号而非单引号；is_postal_search设为false，匹配城市搜索场景。
3. 建立有效会话：通过Session对象维护Cookie，确保请求携带网站要求的会话标识。

内容的提问来源于stack exchange，提问作者robots.txt