Python爬虫仅返回1条结果，预期应为20条的技术求助

问题根源 #

直接通过requests.get获取的是静态HTML，目标网站的股票数据大部分由JavaScript动态加载，静态响应中仅包含1条初始数据；同时原代码的XPath定位错误，把列表外层容器当成了单个股票条目，导致只能抓取到1条结果。

方案一：调用API接口抓取（推荐） #

目标网站通过API接口返回股票数据，直接请求API更高效稳定，无需处理页面渲染。

import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
    'Referer': 'https://www.tickertape.in/screener/equity'
}

# 默认筛选条件，请求20条数据
payload = {
    "sort": [{"sortBy": "marketCap", "sortOrder": "desc"}],
    "page": 1,
    "limit": 20,
    "filters": {},
    "columns": ["name", "subindustry", "marketCap", "closePrice", "pe"]
}

resp = requests.post('https://api.tickertape.in/screener/query', json=payload, headers=headers)
data = resp.json()

all_stocks = []
for item in data['data']['hits']:
    stock = {
        'name': item['name'],
        'subsector': item['subindustry'],
        'marketcap': item['marketCap'],
        'closeprice': item['closePrice'],
        'peratio': item['pe']
    }
    all_stocks.append(stock)

print(all_stocks)
print(f"共抓取到{len(all_stocks)}条股票数据")

方案二：使用Selenium处理动态渲染（适合坚持页面解析的场景） #

通过Selenium加载页面并等待JavaScript渲染完成，再解析完整页面源码。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import html

# 初始化Chrome浏览器（需提前安装ChromeDriver并配置环境变量）
driver = webdriver.Chrome()
driver.get('https://www.tickertape.in/screener/equity')

# 等待股票列表加载完成
WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.XPATH, "//div[@class='jsx-3425801674 screener-row']"))
)

# 获取渲染后的页面源码
page_source = driver.page_source
driver.quit()

tree = html.fromstring(page_source)

# 定位所有股票条目
stocks = tree.xpath("//div[@class='jsx-3425801674 screener-row']")
all_stocks = []

for stock in stocks:
    s = {
        'name': stock.xpath(".//span[contains(@class,'desktop--only pointer')]/text()")[0],
        'subsector': stock.xpath(".//div[contains(@class,'ellipsis-container subindustry')]/span[contains(@class,'desktop--only')]/text()")[0],
        'marketcap': stock.xpath(".//div[contains(@class,'mrktCapf-col')]/span[contains(@class,'data-cell screener-cell desktop--only')]/text()")[0],
        'closeprice': stock.xpath(".//div[contains(@class,'pricef-col')]/span[contains(@class,'data-cell screener-cell desktop--only')]/text()")[0],
        'peratio': stock.xpath(".//div[contains(@class,'apef-col')]/span[contains(@class,'data-cell screener-cell desktop--only')]/text()")[0]
    }
    all_stocks.append(s)

print(all_stocks)
print(f"共抓取到{len(all_stocks)}条股票数据")

内容的提问来源于stack exchange，提问作者Vishal Vasava