如何修改单只股票抓取Python代码实现批量提取股票列表数据并存为DataFrame

import requests
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC

# 配置参数
headers = {'User-Agent': 'Mozilla/5.0'}
stock_list = ['Infosys' , 'Reliance industries', 'wipro' ]
all_data = []

# 初始化浏览器（仅启动一次）
browser = webdriver.Firefox()
browser.get("https://www.tickertape.in/stocks/")
browser.maximize_window()
waiter = wait(browser, 5)

def get_stock_data(stock_name):
    # 定位并清空搜索框
    input_element = waiter.until(EC.visibility_of_element_located((By.ID, 'search-stock-input')))
    input_element.clear()
    input_element.click()
    input_element.send_keys(stock_name)
    input_element.send_keys(Keys.RETURN)
    # 等待页面加载完成
    waiter.until(EC.staleness_of(input_element))
    
    # 直接读取渲染后的页面源码，无需二次请求
    soup = BeautifulSoup(browser.page_source, 'html.parser')
    
    # 提取字段
    try:
        script_name = soup.find(class_ = 'jsx-2256451 security-name').text.strip()
        intrinsic_value = soup.find(class_ = 'jsx-3277407410 jsx-1058798148 lh-138 text-13 commentary-desc').text.strip()
        returns_vs_fd = soup.find(class_ = 'jsx-3947392323 jsx-1058798148 lh-138 text-13 commentary-desc').text.strip()
        divident_returns = soup.find(class_ = 'jsx-566496888 jsx-1058798148 lh-138 text-13 commentary-desc').text.strip()
        entry_point = soup.find(class_ = 'jsx-3697483086 jsx-1058798148 lh-138 text-13 commentary-desc').text.strip()
        red_flag = soup.find(class_ = 'jsx-1920835126 jsx-1058798148 relative no-select tooltip-holder').text.strip()
        red_flag_reason = soup.find(class_ = 'jsx-1920835126 jsx-1058798148 lh-138 text-13 commentary-desc').text.strip()
        
        return {
            'ScriptName': script_name,
            'intrinsic_value': intrinsic_value,
            'Returns_vs_FD_rates': returns_vs_fd,
            'Divident_Returns': divident_returns,
            'Entry_Point': entry_point,
            'Red_Flag_Indicator': red_flag,
            'Red_Flag_Indicator_Reason': red_flag_reason
        }
    except Exception as e:
        print(f"爬取{stock_name}时出错：{str(e)}")
        return None

# 批量爬取
for stock in stock_list:
    data = get_stock_data(stock)
    if data:
        all_data.append(data)

# 生成结果表
df = pd.DataFrame(all_data)
print(df)
# 可选导出到csv
# df.to_csv('stock_data.csv', index=False, encoding='utf-8-sig')

# 关闭浏览器
browser.quit()

核心调整说明 #

• 浏览器实例复用：全程仅启动一次Firefox，无需反复启停，大幅提升爬取效率
• 逻辑封装：单只股票的搜索、数据提取逻辑封装为独立函数，遍历股票列表即可实现批量爬取
• 冗余代码简化：删除重复的列表初始化、循环追加逻辑，直接提取对应字段值存入字典
• 兼容性修复：修正旧版Selenium元素定位语法、原代码缩进错误，适配Selenium 4+版本
• 容错处理：增加异常捕获逻辑，单只股票爬取失败不会中断整个批量任务
• 请求逻辑优化：直接读取Selenium渲染后的页面源码，无需额外调用requests发起二次请求，减少被反爬拦截的概率

内容的提问来源于stack exchange，提问作者user51988