如何用Python抓取URL固定的网站多页数据？附失效代码求助

我编写了三个Python程序，尝试抓取https://www.bseindia.com/markets/debt/TradenSettlement.aspx这个固定网址的多页债务交易结算数据，要求不使用Selenium。此前用同类程序在其他网站可正常抓取，但这三个程序均失效，该网站页数超过10页，希望解决抓取所有页面数据的问题。

程序1 #

import requests
from bs4 import BeautifulSoup
import pandas as pd
import os
import os.path
import datetime
import schedule      
import time

dt = str(datetime.date.today())
today = datetime.datetime.now()
#date_time = today.strftime("%m/%d/%Y_%H_%M_%S")
date_time = today.strftime("%d-%m-%Y_%H.%M")
print("date and time:",date_time)
file_name = 'BSE_Trades_' + date_time


save_path = r"C:\Users\ABCD"              
path = os.path.join(save_path, file_name+".csv")  

url = "https://www.bseindia.com/markets/debt/TradenSettlement.aspx"

#headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52"}


dfs = pd.read_html(url)                                                                                                         
df = dfs[-2]

print(df)

count_row = df.shape[0]

data = {                             
    "__EVENTTARGET" : "",
    '__EVENTARGUMENT': '',  
    "__VIEWSTATE" : "",
    "__VIEWSTATEGENERATOR": "",
    "__VIEWSTATEENCRYPTED": "",
    "__EVENTVALIDATION": "",
    
       }

def updateData(response):
    global data
    soup = BeautifulSoup(response.content, 'html.parser')
    for i in data:
        try:
            data[i] = soup.find("input", id=i).get("value")
        except:
             pass
            
def main(url):
    global data
    targetString = "ctl00$ContentPlaceHolder1$GridViewrcdsFC"
    with requests.Session() as req:
        r = req.get(url)
        df = pd.read_html(r.content, attrs={
                      'id': 'ContentPlaceHolder1_divCT1'}) [0]          

       # 打印第一页表格
        print(df)
        
        # 获取最后一列的最后一个元素，这里存储了总页数
      
        
        try: pageLength = int(df[0][count_row-1][-2])
        except: pageLength=int(df[0][count_row-2][-2])
        else: pageLength = int(df[0][count_row-1][-2])
               

        #try: pageLength = 22
        #except: pageLength = 0
        #else: pageLength = 22
        

        updateData(r)
                
        for pageNumber in range(1,pageLength):
            
            data["__EVENTTARGET"] = targetString #+str(pageNumber)
            data["__EVENTARGUMENT"] = "Page$" + str(pageNumber) #+str(pageNumber)
            data["__VIEWSTATEGENERATOR"] = "1BDEC9B0"
            
            r = req.post(url ,  data=data)
            
            _df = pd.read_html(r.content, attrs={
                          'id': 'ContentPlaceHolder1_divCT1'}) [0]          
            updateData(r)        
            df = df.append(_df)
            print (df)
      
        df.to_csv(path)

main("https://www.bseindia.com/markets/debt/TradenSettlement.aspx")

程序2 #

import json

import pandas as pd
import requests
import datetime
#from datetime import datetime
import os
import os.path
import schedule      
import time


dt = str(datetime.date.today())
today = datetime.datetime.now()
date_time = today.strftime("%d-%m-%Y_%H.%M")
print("date and time:",date_time)
file_name = 'BSE_Data_' + date_time

save_path = r"C:\Users\XYZ"              
path = os.path.join(save_path, file_name+".csv")


endpoint = "https://www.bseindia.com/markets/debt/TradenSettlement.aspx"

headers = {
    #"pageToken": "f06c7498-ac12-4def-95d2-f0fb903fff64",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.52",
    #"X-Requested-With": "XMLHttpRequest",
}

#实际网站里没有Payload，我自己创建了一个

Payload = {
    "columnNames": [
        "Deal Type*(Brokered/Direct/IST)", "ISIN", "Listed/Unlisted security", "Issuer Name", "Coupon (%)",
        "Issue Description", "Traded Price in Rs", "Trade yield (%)", "Yield Type (YTC/YTP/YTM)#",
        "Yield Date", "Trade Value in Rs. Lacs (in face value term)", "Trade Date & Time", "Settlement Date", "Reported trade/Trade executed on RFQ platform",
        "Settlement Status^(Settled/Not Settled/Pending)", "Outside Yield Range",
    ]
}


response = requests.post(endpoint, data=json.dumps(Payload), headers=headers)

df = pd.DataFrame(response.json(), columns=Payload["columnNames"])

print(df)

df.to_csv(path, index=False)

程序3（尝试用Fire Command翻页仍无效） #

import requests
from bs4 import BeautifulSoup
import pandas as pd

data = {
    
    '__EVENTTARGET': "ctl00$ContentPlaceHolder1$GridViewrcdsFC",
    '__EVENTARGUMENT': 'FireCommand:Page$1;PageSize;50',
    '__VIEWSTATE': "",
    '__VIEWSTATEGENERATOR': '1BDEC9B0',
    '__VIEWSTATEENCRYPTED':"",
    '__EVENTVALIDATION':"moMXnWZxj4bZurveAGmOcL0nISClexUE9Z2uw4xpvBOm1MGb2OcWpeoR93Q2hSbZNPFrA13DJe+gToc4zKmJCrAz6mtps/4+Fuc55oo04aW5LAcfpXgJF4F9dtA80NIp6P5vueUYd7iUSQ1sGEnNlWQcghy//kGSS09BCEGrF6iX+zA/9P4X3Yjd8zLJRKyMbAYzKHPVaNNw1QovP7EsqwBhzHWN7R9IjuSvXBwbDC8Gtxkb8JmOx8Uh5ohoih8EmxSjHEjThJY79RbzkRBRzA==",
    
}


def main(url):
    with requests.Session() as req:
        r = req.get(url)
        soup = BeautifulSoup(r.content, 'html.parser')
        data['__VIEWSTATE'] = soup.find("input", id="__VIEWSTATE").get("value")
        data['__EVENTARGUMENT'] = soup.find(
            "input", id="__EVENTARGUMENT").get("value")
        r = req.post(url, data=data)
        df = pd.read_html(r.content, attrs={
                          'id': 'ContentPlaceHolder1_GridViewrcdsFC'})[0]
        df.drop(df.columns[1], axis=1, inplace=True)
        print(df)
        #df.to_csv("data.csv", index=False)


main("https://www.bseindia.com/markets/debt/TradenSettlement.aspx")

内容的提问来源于stack exchange，提问作者New_Coder