使用requests发起带参POST请求CIPC搜索页无法获取预期结果如何解决

问题根因 #

你所访问的是典型ASP.NET WebForm站点，采用UpdatePanel实现局部AJAX刷新，你现有代码提交逻辑没问题，但微软AJAX返回的是特殊分段格式的响应，并非完整HTML，直接打印/解析无法获取结构化内容，需要先提取对应页面片段再处理。

可运行实现代码 #

import requests
from bs4 import BeautifulSoup

link = 'https://eservices.cipc.co.za/Search.aspx'

# 补全请求头避免站点拦截
headers = {
    'X-MicrosoftAjax': 'Delta=true',
    'X-Requested-With': 'XMLHttpRequest',
    'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
    'Accept': '*/*',
    'Referer': 'https://eservices.cipc.co.za/Search.aspx'
}

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    # 首次请求获取动态校验参数和Cookie
    r = s.get(link)
    soup = BeautifulSoup(r.text,"lxml")
    payload = {
        '__VIEWSTATE': soup.select_one("#__VIEWSTATE")["value"],
        '__VIEWSTATEGENERATOR': soup.select_one("#__VIEWSTATEGENERATOR")["value"],
        '__EVENTVALIDATION': soup.select_one("#__EVENTVALIDATION")["value"],
        '__EVENTARGUMENT': '',
        '__LASTFOCUS': '',
        'ctl00$cntMain$ScriptManager1': 'ctl00$cntMain$Updatepanel1|ctl00$cntMain$lnkSearchIcon',
        'ctl00$cntMain$drpSearchOptions': 'EntNo',
        'ctl00$cntMain$txtSearchCIPC': 'K2018288262',
        'ctl00$cntMain$wtmkSearch_ClientState': '',
        '__EVENTTARGET': 'ctl00$cntMain$lnkSearchIcon',
        '__ASYNCPOST': 'true'
    }
    s.headers.update(headers)
    res = s.post(link, data=payload)
    
    # 提取搜索结果HTML片段
    response_parts = res.text.split('|')
    search_result_html = ""
    for i in range(len(response_parts)):
        if response_parts[i] == 'UpdatePanel1':
            search_result_html = response_parts[i+1]
            break
    
    # 解析所需字段
    if search_result_html:
        result_soup = BeautifulSoup(search_result_html, 'lxml')
        # 按照实际需求写选择器提取字段即可，示例打印所有结果行内容
        result_rows = result_soup.select("table tr")
        for row in result_rows:
            cells = row.select("td")
            if cells:
                print([cell.get_text(strip=True) for cell in cells])
    else:
        print("未获取到搜索结果，可替换为浏览器真实User-Agent重试")

内容的提问来源于stack exchange，提问作者MITHU