如何用Python通过POST请求下载网站生成的CSV报告？

你的初始POST请求是有效的——它触发了报告生成流程。网站不会直接返回CSV，因为大型报告是异步生成的。相反，AJAX响应提供了一个URL，用于在报告准备就绪后获取已完成的报告。

修复步骤： #

• 从AJAX响应的JavaScript代码中提取待处理报告的文件名。
• 使用现有会话Cookie发送GET请求到生成的下载URL。
• 将响应内容保存为CSV文件。

修改后的代码： #

import requests
import re

# 你原有的Cookies和Headers
cookies = {
    '_ga': 'GA1.2.938840467.1566745600',
    'terminal': '174224916994986694513353793024390053397',
    '__utmc': '1',
    'request_timing~11947199': '2~2237~74~91~2402',
    '_gid': 'GA1.2.89702438.1657551717',
    'cf_clearance': 'RDmwUSB_b6JmRSJpvrM76reZifV_m6cHjCJ0kmUkAS8-1657566551-0-250',
    'GSG_SESSION_ID': '322919708739472562779456661040511933493',
    'is_session_valid': '1',
    'plack_session': '27e03cd7d13a440955626dbc574adef85a619f88',
    '__utma': '1.938840467.1566745600.1657299681.1657567143.4',
    '__utmz': '1.1657567143.4.4.utmcsr=leetc.com|utmccn=(referral)|utmcmd=referral|utmcct=/',
    '__utmt': '1',
    'session_id': '53da3a29-e3d5-4dd4-96c3-7562a0fb7715',
    '_gat': '1',
    'request_id~1513632513': '6441F026-014E-11ED-9669-AEADB5E8FA7B',
    '__utmb': '1.5.9.1657567169673',
    'request_timing~1513632513': '1~4217~79~105~4401',
}

headers = {
    'authority': 'lee.county-taxes.com',
    'accept': 'application/xml, text/xml, */*; q=0.01',
    'accept-language': 'en-US,en;q=0.9',
    'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'origin': 'https://lee.county-taxes.com',
    'referer': 'https://lee.county-taxes.com/public/reports/real_estate',
    'sec-ch-ua': '"Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
    'x-requested-with': 'XMLHttpRequest',
}

data = 'base_url=public%2Freports%2Freal_estate&parent_request_id=5955C82C-014E-11ED-A791-8C0D896639A2&report_search=cert_status%09%2F%7C%2F%09tax_year%09%3E2014%09roll_year%09*%09cert_sale_date%09%2F%7C%2F%09exemption%09%2F%7C%2F%09deed_status%09%2F%7C%2F%09standard_flag%09%2F%7C%2F%09&report_search_current_user_inputs=&session_id=322919708739472562779456661040511933493&app_url=%2Ftcb%2Fapp&page_url=public%2Freports%2Freal_estate&report_name=Certificate+Information+by+Year&selected_report=624&filetype=csv&delimiter=comma&quoting=on&pdf_report_title=Certificate+Information+by+Year+Report+(Certificate+Search)&add_run_by_to_header=on&paper_size=letter&page_orientation=portrait&page_break=auto&break_selector=tax_year&shade_alternate_rows=on&pdf_report_description=&report_download_email_address=&report_took_too_long_download_email=&displayed_columns=certificate_number%7Ccert_status%7Ctax_year%7Croll_year%7Caccount_number%7Csitus_address%7Ccert_sale_date%7Cissued_date%7Cpurchased_date%7Ccertificate_rate%7Cinterest_start_date%7Cface_amount%7Credeemed_date%7Credemption_amount_paid%7Ctransferred_date%7Cexemption%7Cdeed_status%7Cbidder_number%7Ccertificate_buyer%7Cstandard_flag&hide_results=&sort_by_displayed=&hide_public=&display_name=&current_view=certs&current_display=data&select_view=certs&last_sort_col=1&sbgb_boundary=2&search_order_column1=tax_year&search_order_column2=&select_display=data&search_order_direction1=desc&report_search_dummy=&report_search_dummy=%2F%7C%2F&report_search_dummy=%3E2014&report_search_dummy=&report_search_dummy=&report_search_dummy=%2F%7C%2F&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=&report_search_dummy=%2F%7C%2F&report_search_dummy=%2F%7C%2F&report_search_dummy=&report_search_dummy=&report_search_dummy=%2F%7C%2F&rows_per_page=50&go_to_report_page=&report_page=1&backend=warehouse&print_cover_page=0&download_now=browser&ajax_request=0.4428397794570913&cookie_id=2025887579'

# 步骤1：触发报告生成
response = requests.post('https://lee.county-taxes.com/tcb/app/public/reports/real_estate/download', cookies=cookies, headers=headers, data=data)

# 步骤2：从AJAX响应中提取待处理报告的文件名
response_text = response.text
match = re.search(r'pending_report_file=([a-zA-Z0-9_]+)', response_text)
if not match:
    print("无法提取待处理报告文件名")
    exit()
pending_file = match.group(1)

# 步骤3：获取已生成的报告
download_url = f'https://lee.county-taxes.com/download_pending_report?pending_report_file={pending_file}'
# 更新GET请求的Headers（调整accept以支持CSV）
get_headers = headers.copy()
get_headers['accept'] = 'text/csv,application/xml, text/xml, */*; q=0.01'

csv_response = requests.get(download_url, cookies=cookies, headers=get_headers)

# 步骤4：保存CSV文件
with open('certificate_report.csv', 'wb') as f:
    f.write(csv_response.content)

print("CSV文件下载成功")

注意事项： #

• 会话Cookie：你使用的Cookie与浏览器会话绑定。如果Cookie过期，需要从浏览器开发者工具中重新复制。
• 报告生成延迟：对于大型报告，可能需要在触发生成和获取报告之间添加短暂延迟（使用time.sleep()），给服务器足够的处理时间。
• 速率限制：注意网站的速率限制，避免被封禁。

内容的提问来源于stack exchange，提问作者James McBot