使用Python requests请求Idealista网页返回403错误的问题排查

我正在做一个入门级网页信息爬取项目，用Python的requests库请求Fotocasa网页能正常拿到<Response [200]>，但请求Idealista网页时始终返回<Response [403]>，不确定是不是操作哪里错了。以下是我的测试代码：

测试Fotocasa的代码 #

import requests
from bs4 import BeautifulSoup

url = 'https://www.fotocasa.es/es/comprar/vivienda/valencia-capital/aire-acondicionado-trastero-ascensor-no-amueblado/161485852/d'
# url = 'https://www.idealista.com/inmueble/97795476/'

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'es,es-ES;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'cache-control': 'max-age=0',
    'dnt': '1',
    'sec-ch-ua': '"Chromium";v="106", "Microsoft Edge";v="106", "Not;A=Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.47'
}

r = requests.get(url, headers=headers)
print(r)

req = requests.get(url, headers=headers).text

# 使用BeautifulSoup解析内容
soup = BeautifulSoup(req, "html.parser")

# 获取指定标签信息
inm = soup.find(class_="re-DetailHeader-propertyTitle").text
print(inm)

测试Idealista的代码 #

import requests
from bs4 import BeautifulSoup

# url = 'https://www.fotocasa.es/es/comprar/vivienda/valencia-capital/aire-acondicionado-trastero-ascensor-no-amueblado/161485852/d'
url = 'https://www.idealista.com/inmueble/97795476/'

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'es,es-ES;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'cache-control': 'max-age=0',
    'dnt': '1',
    'sec-ch-ua': '"Chromium";v="106", "Microsoft Edge";v="106", "Not;A=Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.47'
}

r = requests.get(url, headers=headers)
print(r)

req = requests.get(url, headers=headers).text

# 使用BeautifulSoup解析内容
soup = BeautifulSoup(req, "html.parser")

# 获取指定标签信息
inm = soup.find(class_="main-info__title-main").text
print(inm)

Idealista的反爬机制比Fotocasa更严格，403状态码说明请求被服务器判定为非人类访问，可通过以下调整尝试解决：

1. 添加Referer头
   服务器会校验请求来源，直接请求详情页易被拦截。添加Referer字段模拟从Idealista主页跳转：

headers['referer'] = 'https://www.idealista.com/'

2. 用Session维持会话
   先请求Idealista主页获取必要Cookie，再用同一个Session请求详情页，模拟真实用户浏览流程：

session = requests.Session()
# 先访问主页获取Cookie
session.get('https://www.idealista.com/', headers=headers)
# 再请求目标页面
r = session.get(url, headers=headers)
print(r)

3. 简化Headers
   原Headers中部分字段（比如sec-fetch-site: none）会暴露爬虫身份，保留关键字段即可，同时更新User-Agent到较新版本：

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'accept-language': 'es-ES,es;q=0.9',
    'referer': 'https://www.idealista.com/'
}

4. 添加请求延迟
   短时间内重复请求会触发反爬，在请求前添加适当延迟：

import time
time.sleep(2)  # 请求前等待2秒

调整后的完整示例代码 #

import requests
from bs4 import BeautifulSoup
import time

url = 'https://www.idealista.com/inmueble/97795476/'

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'accept-language': 'es-ES,es;q=0.9',
    'referer': 'https://www.idealista.com/'
}

session = requests.Session()
# 先访问主页
session.get('https://www.idealista.com/', headers=headers)
time.sleep(2)

r = session.get(url, headers=headers)
print(r)

if r.status_code == 200:
    req = r.text
    soup = BeautifulSoup(req, "html.parser")
    inm = soup.find(class_="main-info__title-main").text
    print(inm)
else:
    print(f"请求失败，状态码：{r.status_code}")

如果以上方法无效，可能是Idealista更新了反爬策略，入门阶段可先尝试上述基础方案，进阶可考虑用Selenium模拟真实浏览器行为。

内容的提问来源于stack exchange，提问作者Javier Cabrera