使用Python Selenium爬取NFTrade懒加载NFT时仅提取首个元素数据的问题

问题根源 #

1. 提取卡片ID时，使用self.driver.find_element()从整个页面范围查找元素，每次都会返回页面中第一个匹配的Item_itemName__ckoHR元素，而非当前遍历的卡片下的子元素。
2. 提取价格时，XPATH以//开头，会从文档根节点开始全局搜索，忽略了传入的card_data上下文元素，因此同样只会拿到页面中第一个价格元素。

修复后的代码 #

修改get_card_data函数，将元素查找范围限制在当前卡片内，使用相对路径XPATH（以.//开头，表示从当前元素开始向下查找）：

from pprint import pprint
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time

class NFTradeScraper:
    def __init__(self):
        options = Options()
        options.add_argument('--headless')
        options.add_argument('--start-maximized')
        self.driver = webdriver.Chrome(options=options)
        self.wait = WebDriverWait(self.driver, 5)

    def get_latest_card_count(self):
        """Get the count of cards loaded into list of cards."""
        return len(self.driver.find_elements(By.XPATH, '//div[contains(@class, "Item_itemContent__1XIcH")]'))    
    
    def get_cards(self, max_card_count = 200):
        """Extracts and returns card ID and price."""
        URL="https://nftrade.com/collection/zombienft?search=&sort=min_price_asc&contractAddress=0xc031218cef355994d51cda0911b86f0a0e0dccaa&chainName="
        self.driver.get(URL)
        last_card_count = 0
        while last_card_count < max_card_count:
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(3)
            last_card_count = self.get_latest_card_count()
        cards = self.driver.find_elements(By.XPATH, '//div[contains(@class, "Item_itemContent__1XIcH")]')
        return cards
                    
    def get_card_data(self, card_data):
        """Extracts and prints out card specific data."""
        # 从当前card_data元素下查找名称，使用相对路径XPATH
        card_name_element = card_data.find_element(By.XPATH, './/div[contains(@class, "Item_itemName__ckoHR")]')
        card_name = card_name_element.get_attribute("innerHTML")
        card_id = card_name.partition('#')[-1]

        # XPATH开头加.//，限制在当前card_data元素下查找价格
        card_price_element = card_data.find_element(By.XPATH, './/div[contains(@class, "Item_itemPriceValueTxt__lblqJ")]')
        card_price = card_price_element.get_attribute("innerHTML")
        return {
            'id': int(card_id), 'price': card_price
            }

if __name__ == '__main__':
   scraper = NFTradeScraper();
   cards = scraper.get_cards(max_card_count=100)
   card_data = []
   for card in cards:
    info = (scraper.get_card_data(card)) 
    card_data.append(info)
       
pprint(card_data)

额外优化建议 #

替换固定的time.sleep(3)为显式等待，监听卡片数量变化，提升爬取效率和稳定性：

# 在get_cards函数中替换原while循环部分
while last_card_count < max_card_count:
    self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    # 等待卡片数量增加，最多等待5秒
    try:
        self.wait.until(lambda d: len(d.find_elements(By.XPATH, '//div[contains(@class, "Item_itemContent__1XIcH")]')) > last_card_count)
    except:
        # 若等待后数量未增加，说明已加载完所有卡片，退出循环
        break
    last_card_count = self.get_latest_card_count()

内容的提问来源于stack exchange，提问作者PaigeN11