如何改进Scrapy代码爬取JS渲染的Electro Planet iPhone分页页面？

问题背景 #

作为爬虫新手，我尝试爬取https://www.electroplanet.ma/网站上的所有iPhone商品，目前编写的Scrapy代码需要手动在start_urls中写入各分页链接，因为网站采用JavaScript实现分页，这显然不是最佳实践，恳请提供代码改进建议。

现有代码 #

import scrapy 
from ..items import EpItem
# 注意：原代码中parse_item使用了re但未导入，需补充import re

class ep(scrapy.Spider):
    name = "ep"
    start_urls = [
        "https://www.electroplanet.ma/smartphone-tablette-gps/smartphone/iphone?p=1",
        "https://www.electroplanet.ma/smartphone-tablette-gps/smartphone/iphone?p=2"
    ]

    def parse(self, response):
        products = response.css("ol li")  # 选择器过于宽泛，可能包含非商品元素
        for product in products :
            try:
                lien = product.css("a.product-item-link::attr(href)").get()
                image= product.css("a.product-item-photo::attr(href)").get()
                yield response.follow(lien, callback=self.parse_item, cb_kwargs={"image":image}) 
            except: 
                pass     

    def parse_item(self, response, image):
        item = EpItem()
        item["Nom"]= response.css(".ref::text").get()
        pattern = re.compile(r"\s*(\S+(?:\s+\S+)*)\s*")
        item["Catégorie"]= pattern.search(response.xpath("//h1/a/text()").get()).group(1)
        item["Marque"]=pattern.search(response.xpath("//*[@data-th='Marque']/text()").get()).group(1)
        try :
            item["RAM"]= pattern.search(response.xpath("//*[@data-th='MÉMOIRE RAM']/text()").get()).group(1)
        except:
            pass
        item["ROM"]=pattern.search(response.xpath("//*[@data-th='MÉMOIRE DE STOCKAGE']/text()").get()).group(1)
        item["Couleur"]=pattern.search(response.xpath("//*[@data-th='COULEUR']/text()").get()).group(1)
        item["lien"]=response.request.url
        item["image"]=image
        item["état"]="neuf"
        item["Market"]= "Electro Planet"
        yield item

代码改进建议 #

1. 优化基础代码细节

• 精准选择器：将商品列表选择器从ol li改为ol.products-list li.item.product.product-item，避免选中分页栏、侧边栏等无关元素。
• 补充依赖导入：在代码开头添加import re，否则parse_item方法会因未识别re模块报错。
• 细化异常处理：替换宽泛的except:，捕获具体异常（如AttributeError），同时增加元素存在性判断，减少无效请求：

  try:
      lien = product.css("a.product-item-link::attr(href)").get()
      image = product.css("img.product-image-photo::attr(src)").get()  # 直接取图片src更准确
      if lien and image:
          yield response.follow(lien, callback=self.parse_item, cb_kwargs={"image": image})
  except AttributeError as e:
      self.logger.error(f"处理商品时出错: {e}")
  

2. 自动处理JavaScript分页（推荐方案：Scrapy-Playwright渲染）

针对JS渲染的分页，使用scrapy-playwright中间件模拟浏览器行为，自动识别并点击下一页，无需手动构造URL：

步骤1：安装依赖

pip install scrapy-playwright

步骤2：配置项目（修改settings.py）

# 启用Playwright下载处理器
DOWNLOAD_HANDLERS = {
    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}

# 配置Playwright无头模式
PLAYWRIGHT_LAUNCH_OPTIONS = {
    "headless": True,
    "timeout": 30000,
}

# 启用Playwright中间件
SPIDER_MIDDLEWARES = {
    "scrapy_playwright.middleware.PlaywrightMiddleware": 543,
}

步骤3：修改爬虫代码实现自动翻页

import scrapy
import re
from ..items import EpItem

class EpSpider(scrapy.Spider):
    name = "ep"
    start_urls = ["https://www.electroplanet.ma/smartphone-tablette-gps/smartphone/iphone"]

    def start_requests(self):
        for url in self.start_urls:
            # 标记请求需要Playwright渲染
            yield scrapy.Request(
                url,
                meta={"playwright": True, "playwright_include_page": True},
                callback=self.parse
            )

    async def parse(self, response):
        page = response.meta["playwright_page"]
        
        # 解析当前页商品
        products = response.css("ol.products-list li.item.product.product-item")
        for product in products:
            try:
                lien = product.css("a.product-item-link::attr(href)").get()
                image = product.css("img.product-image-photo::attr(src)").get()
                if lien and image:
                    yield response.follow(lien, callback=self.parse_item, cb_kwargs={"image": image})
            except AttributeError as e:
                self.logger.error(f"解析商品失败: {e}")

        # 自动点击下一页
        try:
            next_button = page.locator("li.item.pages-item-next a")
            if await next_button.count() > 0:
                await next_button.click()
                await page.wait_for_load_state("networkidle")  # 等待页面加载完成
                # 发起下一页的渲染请求
                yield scrapy.Request(
                    page.url,
                    meta={"playwright": True, "playwright_include_page": True},
                    callback=self.parse
                )
            else:
                self.logger.info("已爬取所有分页")
        except Exception as e:
            self.logger.error(f"翻页失败: {e}")
        finally:
            await page.close()

    def parse_item(self, response, image):
        item = EpItem()
        # 增加默认值与strip处理，避免空值或冗余空格
        item["Nom"] = response.css(".ref::text").get(default="").strip()
        
        category_text = response.xpath("//h1/a/text()").get(default="")
        item["Catégorie"] = re.search(r"\s*(\S+(?:\s+\S+)*)\s*", category_text).group(1) if category_text else ""
        
        marque_text = response.xpath("//*[@data-th='Marque']/text()").get(default="")
        item["Marque"] = re.search(r"\s*(\S+(?:\s+\S+)*)\s*", marque_text).group(1) if marque_text else ""
        
        ram_text = response.xpath("//*[@data-th='MÉMOIRE RAM']/text()").get(default="")
        item["RAM"] = re.search(r"\s*(\S+(?:\s+\S+)*)\s*", ram_text).group(1) if ram_text else ""
        
        rom_text = response.xpath("//*[@data-th='MÉMOIRE DE STOCKAGE']/text()").get(default="")
        item["ROM"] = re.search(r"\s*(\S+(?:\s+\S+)*)\s*", rom_text).group(1) if rom_text else ""
        
        couleur_text = response.xpath("//*[@data-th='COULEUR']/text()").get(default="")
        item["Couleur"] = re.search(r"\s*(\S+(?:\s+\S+)*)\s*", couleur_text).group(1) if couleur_text else ""
        
        item["lien"] = response.request.url
        item["image"] = image
        item["état"] = "neuf"
        item["Market"] = "Electro Planet"
        yield item

3. 备选方案：分析分页API（高效无渲染）

打开浏览器开发者工具「网络」面板，滚动到分页区域，查看是否有XHR请求返回JSON格式的商品数据。若找到此类接口（如https://www.electroplanet.ma/api/products?category=iphone&page=1），可直接构造分页URL循环请求，无需渲染页面，效率更高。

内容的提问来源于stack exchange，提问作者Aysr