如何在Python中从HTML响应体提取CSRF_NONCE与jsessionid

需要在Python中从HTML响应体里提取首次出现的CSRF_NONCE和jsessionid值，响应中这两个值重复出现36次且内容一致。

响应片段示例 #

<td class="row-left"><a href="/manager/html/list;jsessionid=A9B9660B7F3B1238FE0037843C225537?org.apache.catalina.filters.CSRF_NONCE=2AF805C8EF13A9B39258B056FF37136C">List Applications</a></td>

<td class="row-center"><a href="/manager/../docs/html-manager-howto.html" rel="noopener noreferrer">HTML Manager Help</a></td>

<td class="row-center"><a href="/manager/../docs/manager-howto.html" rel="noopener noreferrer">Manager Help</a></td>

<td class="row-right"><a href="/manager/status;jsessionid=A9B9660B7F3B1238FE0037843C225537?org.apache.catalina.filters.CSRF_NONCE=2AF805C8EF13A9B39258B056FF37136C">Server Status</a></td>

现有请求代码 #

from jproperties import Properties
import requests
from autorizationModule import *

configs = Properties()
with open('app-config', 'rb') as config_file:
    configs.load(config_file)
# 用户名和密码从配置文件读取
TOKEN = authorization(configs.get("user_name").data, configs["password"].data)
print(f"Token Value", TOKEN)
def fetchCSRFNonce():
    BASE_URL =  configs.get("app_url").data
    payload = {}
    HEADER_VALUE = {
        "Authorization" : "Basic "+str(TOKEN)
    }
    response = requests.request("GET", BASE_URL, headers=HEADER_VALUE, data=payload)
    print(response.content)
fetchCSRFNonce()

方法一：正则表达式提取（简单直接） #

利用正则匹配固定格式的jsessionid和CSRF_NONCE，re.search会返回首次匹配结果，符合需求。修改fetchCSRFNonce函数如下：

import re

def fetchCSRFNonce():
    BASE_URL =  configs.get("app_url").data
    payload = {}
    HEADER_VALUE = {
        "Authorization" : "Basic "+str(TOKEN)
    }
    response = requests.request("GET", BASE_URL, headers=HEADER_VALUE, data=payload)
    html_content = response.text
    
    # 匹配32位十六进制格式的jsessionid
    jsessionid_match = re.search(r'jsessionid=([0-9A-F]{32})', html_content)
    # 匹配32位十六进制格式的CSRF_NONCE
    csrf_nonce_match = re.search(r'org.apache.catalina.filters.CSRF_NONCE=([0-9A-F]{32})', html_content)
    
    if jsessionid_match and csrf_nonce_match:
        jsessionid = jsessionid_match.group(1)
        csrf_nonce = csrf_nonce_match.group(1)
        print(f"提取到的jsessionid: {jsessionid}")
        print(f"提取到的CSRF_NONCE: {csrf_nonce}")
        return jsessionid, csrf_nonce
    else:
        print("未找到目标值")
        return None, None

方法二：BeautifulSoup解析HTML（更健壮） #

如果HTML结构可能变动，用专业解析库更可靠。先安装依赖：pip install beautifulsoup4，再修改代码：

from bs4 import BeautifulSoup
from urllib.parse import urlparse, parse_qs

def fetchCSRFNonce():
    BASE_URL =  configs.get("app_url").data
    payload = {}
    HEADER_VALUE = {
        "Authorization" : "Basic "+str(TOKEN)
    }
    response = requests.request("GET", BASE_URL, headers=HEADER_VALUE, data=payload)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 遍历所有带href属性的a标签，找到同时包含两个目标值的链接
    for a_tag in soup.find_all('a', href=True):
        href = a_tag['href']
        if 'jsessionid' in href and 'CSRF_NONCE' in href:
            # 拆分提取jsessionid
            jsession_part = href.split(';', 1)[1]
            jsessionid = jsession_part.split('=', 1)[1].split('?', 1)[0]
            # 解析URL参数提取CSRF_NONCE
            query_params = parse_qs(urlparse(href).query)
            csrf_nonce = query_params['org.apache.catalina.filters.CSRF_NONCE'][0]
            
            print(f"提取到的jsessionid: {jsessionid}")
            print(f"提取到的CSRF_NONCE: {csrf_nonce}")
            return jsessionid, csrf_nonce
    print("未找到目标值")
    return None, None

内容的提问来源于stack exchange，提问作者Jyoti Prakash Mallick