网页抓取与POST请求问题：无法获取预期结构化数据

问题描述 #

我用Python的requests和BeautifulSoup编写网页抓取脚本，目标是从https://letmepost.com/check-da-pa获取指定域名的DA（域名权威值）、PA（页面权威值）、垃圾邮件评分、域名年龄、过期时间及IP地址。请求能成功响应，但仅返回域名“cnn.com”，无法获取预期的结构化数据。附上我的代码，恳请提供修正或优化建议：

import requests
from bs4 import BeautifulSoup

import re

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

session = requests.Session() 
r = session.get('https://letmepost.com/check-da-pa', headers=headers)

soup = BeautifulSoup(r.content, 'html.parser')

script_tag = soup.find("script", string=lambda text: text and "X-CSRF-TOKEN" in text)

if script_tag:
    token_pattern = r"'X-CSRF-TOKEN': '(.*?)'"
    csrf_token_match = re.search(token_pattern, script_tag.string)

    if csrf_token_match:
        csrf_token = csrf_token_match.group(1)
        print(f"CSRF Token: {csrf_token}")
    else:
        print("CSRF Token pattern not found in script content.")
else:
    print("Script tag with 'X-CSRF-TOKEN' not found.")


data = "prothomalo.com"

data_payload = {
    "domains ": data,
    "X-CSRF-TOKEN": csrf_token
}

# Existing data_payload
data_payload = {
    "X-CSRF-TOKEN": csrf_token
}

# Adding new data to data_payload
data_payload["listen"] = "4OdeyKA6Sku1ff86NqkAibxXGf4kfOQUcbepB84U"
data_payload["secret"] = "PNxcg1LI9331xX7"
data_payload["urls"] = "prothomalo.com"
data_payload["fromBrowser"] = "1705462709623"
data_payload["timestamp"] = "1705462686"
data_payload["direct"] = True  # Assuming you want to add a boolean False, not the string 'false'

# The data_payload now contains the additional data
 
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "X-CSRF-TOKEN": csrf_token
}

url = "https://letmepost.com/check-da-pa"

response = session.post(url, data=data_payload, headers=headers)

print(response.status_code)

import pandas as pd
result = BeautifulSoup(response.content, 'html.parser')
print(result)

问题分析与修正方案 #

你的代码存在几个核心问题：请求地址错误、硬编码动态参数、请求格式不匹配、数据解析方式错误，以下是修正后的完整代码及说明：

修正后的代码 #

import requests
from bs4 import BeautifulSoup
import re
import time

# 基础请求头，模拟浏览器AJAX请求
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "application/json, text/javascript, */*; q=0.01",
    "X-Requested-With": "XMLHttpRequest"
}

session = requests.Session()

# 1. 获取页面，提取动态参数和CSRF Token
page_response = session.get('https://letmepost.com/check-da-pa', headers=headers)
soup = BeautifulSoup(page_response.content, 'html.parser')

# 从meta标签提取CSRF Token（比正则匹配脚本更可靠）
csrf_token = soup.find('meta', {'name': 'csrf-token'})['content']
print(f"CSRF Token: {csrf_token}")

# 提取页面动态生成的listen和secret参数
listen = None
secret = None
for script in soup.find_all('script'):
    if script.string:
        listen_match = re.search(r"listen: '(.*?)'", script.string)
        secret_match = re.search(r"secret: '(.*?)'", script.string)
        if listen_match:
            listen = listen_match.group(1)
        if secret_match:
            secret = secret_match.group(1)
        if listen and secret:
            break

if not listen or not secret:
    print("无法提取listen或secret参数")
    exit()

# 2. 准备请求参数，动态生成时间戳
target_domain = "prothomalo.com"
current_timestamp = str(int(time.time()))  # 秒级时间戳
from_browser = str(int(time.time() * 1000))  # 毫秒级时间戳

data_payload = {
    "listen": listen,
    "secret": secret,
    "urls": target_domain,
    "fromBrowser": from_browser,
    "timestamp": current_timestamp,
    "direct": True
}

# 更新请求头，添加CSRF Token和JSON格式标识
headers.update({
    "X-CSRF-TOKEN": csrf_token,
    "Content-Type": "application/json; charset=UTF-8"
})

# 3. 请求正确的API接口（不是页面地址）
api_url = "https://letmepost.com/process-check-da-pa"
api_response = session.post(api_url, json=data_payload, headers=headers)

print(f"响应状态码: {api_response.status_code}")

# 4. 解析JSON格式的响应数据
if api_response.status_code == 200:
    result_data = api_response.json()
    if result_data.get('success'):
        print("\n抓取到的域名信息：")
        for item in result_data.get('data', []):
            print(f"域名: {item.get('domain')}")
            print(f"DA值: {item.get('da')}")
            print(f"PA值: {item.get('pa')}")
            print(f"垃圾邮件评分: {item.get('spam_score')}")
            print(f"域名年龄: {item.get('domain_age')}")
            print(f"过期时间: {item.get('expire_date')}")
            print(f"IP地址: {item.get('ip')}")
            print("---")
else:
    print(f"请求失败，响应内容: {api_response.text}")

关键修正点说明 #

• 请求地址修正：实际数据查询接口是https://letmepost.com/process-check-da-pa，而非页面地址check-da-pa
• 动态参数提取：listen、secret是页面脚本实时生成的，不能硬编码，需从页面中提取
• 请求格式调整：接口要求JSON格式的请求体，需用json=data_payload而非data=data_payload
• 数据解析优化：接口返回JSON格式数据，直接用response.json()解析即可，无需BeautifulSoup
• 请求头完善：添加X-Requested-With: XMLHttpRequest模拟AJAX请求，否则服务器会拒绝非页面请求

内容的提问来源于stack exchange，提问作者Humayun Ahmad Rajib