如何用Python获取无POST接口网站的列表数据？

我想爬取网站https://gmail.inputekno.com/上的列表数据，但检查后发现该网站没有提供POST请求接口，尝试了以下Python代码却失败了：

import requests

cookies = {
    '_ga': 'GA1.1.1869494453.1672283765',
    '__gads': 'ID=e350f661fb3c1b6a-22c8b78c11d900b8:T=1672283764:RT=1672283764:S=ALNI_MYcfleQdj417a3BQakIyzzrp83MdQ',
    '__gpi': 'UID=00000b9a196b924e:T=1672283764:RT=1672283764:S=ALNI_MZ4UaRvjE4GzR-k0Na5Jj-HBksD4w',
    '_ga_R3D1879B9V': 'GS1.1.1672283764.1.1.1672284364.0.0.0',
}

headers = {
    'authority': 'gmail.inputekno.com',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-language': 'en-US,en;q=0.9',
    'cache-control': 'no-cache',
    # 'cookie': '_ga=GA1.1.1869494453.1672283765; __gads=ID=e350f661fb3c1b6a-22c8b78c11d900b8:T=1672283764:RT=1672283764:S=ALNI_MYcfleQdj417a3BQakIyzzrp83MdQ; __gpi=UID=00000b9a196b924e:T=1672283764:RT=1672283764:S=ALNI_MZ4UaRvjE4GzR-k0Na5Jj-HBksD4w; _ga_R3D1879B9V=GS1.1.1672283764.1.1.1672284364.0.0.0',
    'pragma': 'no-cache',
    'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
    'sec-ch-ua-arch': '""',
    'sec-ch-ua-bitness': '"64"',
    'sec-ch-ua-full-version-list': '"Not?A_Brand";v="8.0.0.0", "Chromium";v="108.0.5359.125", "Google Chrome";v="108.0.5359.125"',
    'sec-ch-ua-mobile': '?1',
    'sec-ch-ua-model': '"Nexus 5"',
    'sec-ch-ua-platform': '"Android"',
    'sec-ch-ua-platform-version': '"6.0"',
    'sec-ch-ua-wow64': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36',
}

payload : { 'username':'danielmantha'}
response = requests.post('https://gmail.inputekno.com/', cookies=cookies, headers=headers,data=payload)

可行的解决思路 #

1. 直接解析静态HTML页面 #

如果网站列表数据是直接渲染在页面源码中的，无需异步加载，可改用GET请求获取页面后，用HTML解析库提取数据：

import requests
from bs4 import BeautifulSoup

cookies = {
    '_ga': 'GA1.1.1869494453.1672283765',
    '__gads': 'ID=e350f661fb3c1b6a-22c8b78c11d900b8:T=1672283764:RT=1672283764:S=ALNI_MYcfleQdj417a3BQakIyzzrp83MdQ',
    '__gpi': 'UID=00000b9a196b924e:T=1672283764:RT=1672283764:S=ALNI_MZ4UaRvjE4GzR-k0Na5Jj-HBksD4w',
    '_ga_R3D1879B9V': 'GS1.1.1672283764.1.1.1672284364.0.0.0',
}

headers = {
    'authority': 'gmail.inputekno.com',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-language': 'en-US,en;q=0.9',
    'cache-control': 'no-cache',
    'pragma': 'no-cache',
    'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
    'sec-ch-ua-arch': '""',
    'sec-ch-ua-bitness': '"64"',
    'sec-ch-ua-full-version-list': '"Not?A_Brand";v="8.0.0.0", "Chromium";v="108.0.5359.125", "Google Chrome";v="108.0.5359.125"',
    'sec-ch-ua-mobile': '?1',
    'sec-ch-ua-model': '"Nexus 5"',
    'sec-ch-ua-platform': '"Android"',
    'sec-ch-ua-platform-version': '"6.0"',
    'sec-ch-ua-wow64': '?0',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'none',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36',
}

# 改用GET请求获取完整页面
response = requests.get('https://gmail.inputekno.com/', cookies=cookies, headers=headers)
if response.status_code == 200:
    soup = BeautifulSoup(response.text, 'html.parser')
    # 根据页面实际结构修改选择器，比如定位列表项标签
    list_items = soup.find_all('div', class_='your-list-item-class')
    for item in list_items:
        print(item.get_text(strip=True))

2. 模拟浏览器动态渲染 #

如果数据是通过JavaScript异步加载的，可使用浏览器自动化工具模拟页面加载过程，再提取数据：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time

chrome_options = Options()
chrome_options.add_argument('--headless=new')  # 无头模式，不显示浏览器窗口
chrome_options.add_argument('--user-agent=Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36')

driver = webdriver.Chrome(options=chrome_options)
driver.get('https://gmail.inputekno.com/')

# 等待页面加载完成，可根据实际情况调整等待时间或改用显式等待
time.sleep(3)

# 根据页面实际结构提取列表数据
list_items = driver.find_elements(By.CLASS_NAME, 'your-list-item-class')
for item in list_items:
    print(item.text.strip())

driver.quit()

3. 检查GET参数或隐藏接口 #

• 打开浏览器开发者工具（F12），切换到Network标签，刷新页面，查看所有GET请求的参数，尝试在URL中添加筛选参数（如?username=danielmantha）获取目标数据
• 筛选XHR/Fetch请求，检查是否有未发现的JSON接口，直接请求该接口获取结构化数据

内容的提问来源于stack exchange，提问作者AJil Richardo