使用Scrapy+scrapy-splash爬取Woolworths站点无法获取JS渲染的图片与价格求助

1. Splash等待时长不足，未等页面异步接口返回数据完成渲染就返回了HTML
2. 你使用的CSS选择器与站点实际元素结构不匹配
3. 依赖的scrapyjs包为旧版本废弃组件，会影响Splash功能正常运行

1. 升级依赖包 #

卸载旧版本的scrapyjs，安装最新版scrapy-splash

pip uninstall scrapyjs
pip install -U scrapy-splash

2. 修正settings.py配置 #

确保配置项如下，删除旧的scrapyjs相关配置：

SPLASH_URL = 'http://localhost:8050'
DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}
SPIDER_MIDDLEWARES = {
    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
# 可按需关闭robots协议校验
ROBOTSTXT_OBEY = False

3. 修改爬虫代码 #

新增Lua渲染脚本，确保元素加载完成 #

在爬虫类中新增渲染脚本，确保价格、图片元素加载完成再返回页面内容：

import scrapy
import json
from scrapy_splash import SplashRequest

# 渲染脚本，等待目标元素加载完成
RENDER_SCRIPT = """
function main(splash, args)
  splash:set_user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36")
  assert(splash:go(args.url))
  -- 等待页面基础渲染
  splash:wait(3)
  -- 轮询等待价格元素出现，最长等待10秒
  splash:wait_for_resume([[
    function main(splash) {
      const timer = setInterval(() => {
        if (document.querySelector('div.product-price__primary, script[type="application/ld+json"]')) {
          clearInterval(timer);
          splash.resume();
        }
      }, 500);
    }
  ]], 10)
  return splash:html()
end
"""

class Myspider(scrapy.Spider):
    name = 'WooliesSpider'
    base_url = 'https://www.woolworths.co.za'
    
    def start_requests(self):
        urls = [
            'https://www.woolworths.co.za/dept/Food/_/N-1z13sk5'
        ]
        for url in urls:
            yield SplashRequest(url=url, callback=self.parse)

    def parse(self,response):
        containers = response.css('div.lazyload-container.landing__block.landing__block--half-fourth')
        link_tags = [c.css('a.landing__link') for c in containers]
        link = link_tags[0]
        url = self.base_url+link.attrib['href']
        print(url)
        yield SplashRequest(url,callback=self.parse_product_page)

    def parse_product_page(self,response):
        print('parse_product_page')
        items = response.css('div.product-list__item')
        item = items[0]
        link = item.css('a.product--view').attrib['href']
        url = self.base_url+link
        print(url)
        # 改用execute端点传入lua脚本
        yield SplashRequest(
            url=url,
            callback=self.parse_item_page,
            endpoint='execute',
            args={
                'lua_source': RENDER_SCRIPT,
                'timeout': 20
            }
        )
    
    def parse_item_page(self,response):
        print('*****parse_item_page*****')
        price = None
        image = None
        # 优先提取页面内嵌的结构化数据，稳定性最高
        ld_raw = response.xpath('//script[@type="application/ld+json"]/text()').get()
        if ld_raw:
            try:
                ld_data = json.loads(ld_raw)
                price = ld_data.get('offers', {}).get('price')
                image = ld_data.get('image')
                print(f'结构化数据提取价格：{price}')
                print(f'结构化数据提取主图：{image}')
            except json.JSONDecodeError:
                pass
        # 兜底CSS提取方案
        if not price:
            price = response.css('div.product-price__primary ::text').get().strip()
        if not image:
            image = response.css('div.product-carousel__item img::attr(src)').get()
        print(f'最终提取价格：{price}')
        print(f'最终提取图片：{image}')

内容的提问来源于stack exchange，提问作者Thabiso Shezi