使用Requests与BeautifulSoup爬取NSW Strata地址失败，求解决方案

我需要基于NSW政府的分层计划搜索页面，用requests和BeautifulSoup写脚本完成以下操作：选择「Strata plan number」选项，输入11并搜索，爬取结果中的地址。但运行现有脚本后，无法获取目标地址，代码如下：

import re
import requests
from bs4 import BeautifulSoup

link = 'https://www.nsw.gov.au/housing-and-construction/strata/strata-search'
url = 'https://www.stratahub.nsw.gov.au/prweb/PRAuth/app/ssr_4380/6nxCgYjOTS_fVOVfeekVPA*/!SchemeSearch?pzTransactionId=cc5ddc1ecec1c095231675db14450f87&pzFromFrame=&pzPrimaryPageName=pyDisplayHarness&AJAXTrackID=22'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
    'accept': '*/*',
    'accept-encoding': 'gzip, deflate, br, zstd',
    'accept-language': 'en-US,en;q=0.9',
    'X-Requested-With': 'XMLHttpRequest',
    'origin': 'https://www.stratahub.nsw.gov.au',
}

payload = {
    "$PSchemeSearch$pSearchBy": "Strata Plan Number",
    "$PSchemeSearch$pSchemePlanNumber": 11,
    "pzuiactionzzz": "",
    "PreActivitiesList": "",
    "sectionParam": "",
    "ActivityParams": "=",
    "$ODesktopWrapperInclude": "",
    "$ODeterminePortalTop": "",
    "$ODynamicLayout": "",
    "$ODynamicLayoutCell": "",
    "$OEvalDOMScripts_Include": "",
    "$OForm": "",
    "$OHarness": "",
    "$OHarnessStaticJSEnd": "",
    "$OHarnessStaticJSStart": "",
    "$OHarnessStaticScriptsClientValidation": "",
    "$OPMCHarnessStaticScripts": "",
    "$OSessionUser": "",
    "$OSurveyStaticScripts": "",
    "$OWorkformStyles": "",
    "$OpxAutoComplete": "",
    "$OpxButton": "",
    "$OpxDisplayText": "",
    "$OpxHarnessContent": "",
    "$OpxLayoutContainer": "",
    "$OpxNonTemplate": "",
    "$OpxRadioButtons": "",
    "$OpxSection": "",
    "$OpxVisible": "",
    "$OpxWorkArea": "",
    "$OpxWorkAreaContent": "",
    "$OpyDirtyCheckConfirm": "",
    "$OpyWorkFormStandardEnd": "",
    "$OpyWorkFormStandardStart": "",
    "$OpzAutoCompleteAGIncludes": "",
    "$OpzHarnessInlineScriptsEnd": "",
    "$OpzHarnessInlineScriptsStart": "",
    "$OpzPortalFavIcon": "",
    "$OpzPortalIcon": "",
    "$Opzpega_ui_harnesscontext": "",
    "$Opzpega_web_mashup": "",
    "$OpxTextInput": "",
    "$OpzDecimalInclude": "",
    "pyEncodedParameters": True,
    "pzKeepPageMessages": False,
    "strPHarnessClass": "Data-Portal",
    "strPHarnessPurpose": "SearchStrataScheme",
    "UITemplatingStatus": "Y",
    "StreamName": "SchemeSearch",
    "BaseReference": "SchemeSearch",
    "bClientValidation": True,
    "FormError": "NONE",
    "pyCustomError": "DisplayErrors",
    "UsingPage": True,
    "HeaderButtonSectionName": "-1",
    "PagesToRemove": "",
    "pzHarnessID": "HID387D2E2FCEE4EC200B5BAEA8C6A5D859",
    "inStandardsMode": True
}

with requests.Session() as s:
    s.headers.update(headers)
    res = s.get(link)
    soup = BeautifulSoup(res.text,"lxml")
    code_url = soup.select_one("iframe[title='Strata Search Production']")['data-src']
    s.headers['referer'] = code_url
    payload['pzuiactionzzz'] = code_url.split("?")[-1]
    r = s.post(url,data=payload)
    print(r.status_code)
    print(r.text)

原代码核心问题是硬编码了动态生成的请求URL，且未正确从iframe页面提取必要的会话参数和提交地址。以下是修改后的脚本：

import requests
from bs4 import BeautifulSoup

# 主页面URL
main_url = 'https://www.nsw.gov.au/housing-and-construction/strata/strata-search'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
}

with requests.Session() as session:
    # 1. 访问主页面，获取iframe的链接
    main_response = session.get(main_url, headers=headers)
    main_soup = BeautifulSoup(main_response.text, 'lxml')
    iframe_src = main_soup.select_one("iframe[title='Strata Search Production']")['data-src']
    
    # 2. 访问iframe页面，获取表单提交地址
    iframe_response = session.get(iframe_src, headers=headers)
    iframe_soup = BeautifulSoup(iframe_response.text, 'lxml')
    
    # 提取表单的action URL并拼接完整地址
    form_action = iframe_soup.select_one("form#pyForm")['action']
    submit_url = f"https://www.stratahub.nsw.gov.au{form_action}"
    
    # 3. 构造简化的请求payload，只保留关键参数
    payload = {
        "$PSchemeSearch$pSearchBy": "Strata Plan Number",
        "$PSchemeSearch$pSchemePlanNumber": "11",
        "pzSubmitAction": "Search"  # 触发搜索的关键参数
    }
    
    # 4. 提交搜索请求并解析结果
    search_response = session.post(submit_url, data=payload, headers=headers)
    search_soup = BeautifulSoup(search_response.text, 'lxml')
    
    # 5. 提取地址信息
    address_elements = search_soup.select(".address-line")
    if address_elements:
        print("找到的地址：")
        for elem in address_elements:
            print(elem.get_text(strip=True))
    else:
        print("未找到地址信息")

修改要点说明 #

• 移除硬编码的动态URL，改为从iframe页面的表单中提取action属性作为提交地址，保证URL的有效性。
• 简化payload，只保留触发搜索的核心参数（搜索类型、搜索值、提交动作），多余参数会被服务器自动忽略。
• 用同一个requests.Session()实例处理所有请求，自动维护会话状态和cookies。
• 将搜索值调整为字符串类型，避免类型不匹配导致的服务器校验错误。

内容的提问来源于stack exchange，提问作者robots.txt