Scrapy爬取artofmanliness站点可输出URL但无提取数据问题求助

原问题翻译 #

我正尝试从目标站点提取文章相关信息，作为Scrapy新手目前遇到的问题是：可以正常输出所有正确的文章URL，但无法得到任何提取后的字段数据，暂时找不到代码中遗漏或需要调整的部分，求相关帮助。

原spider代码 #

import scrapy
from scrapy.http import Request

class ArticlesSpider(scrapy.Spider):
    name = 'articles'
    allowed_domains = ['artofmanliness.com']
    max_pages = 200

    def start_requests(self):
        for i in range(self.max_pages):
            yield scrapy.Request('http://artofmanliness.com/articles/page/%d/' % i, callback=self.parse)

    def parse(self, response):
        # AOM有文章列表，每页约189篇
        for article in response.xpath('//article[contains(@class, "aom-article-simple")]'):
            url = article.xpath('.//a/@href').extract()
            print(url)

            if url:
                yield Request(url=url[0], callback=self.parse_article)

    def parse_article(self, response):
        title = response.xpath('//*[@id="post-title entry-title"]/header/h1//text()').extract()
        category = response.xpath('//*[@id="in-category"]/header/p[1]//text()').extract()
        date = response.xpath('//*[@id="single-date"]/header/p[2]/span[2]//text()').extract()

        yield {
            'Title': title,
            'Category': category,
            'Date': date,
            'URL': response.url           
        }

原settings.py配置 #

BOT_NAME = 'aom'    
SPIDER_MODULES = ['aom.spiders']
NEWSPIDER_MODULE = 'aom.spiders'


# 配置User-Agent避免被反爬拦截
USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36"

# 不遵循robots.txt规则
ROBOTSTXT_OBEY = False
HTTPERROR_ALLOW_ALL = True

问题根因 #

• 详情页XPath选择器规则与站点实际DOM结构不匹配：代码误将元素的class属性值作为id属性检索，且元素层级路径完全错误
• 列表页页码遍历从0开始，会请求不存在的/page/0/路径
• 列表页请求使用已废弃的http协议，虽支持自动跳转但存在潜在异常风险

修正后可运行代码 #

爬虫代码 #

import scrapy
from scrapy.http import Request

class ArticlesSpider(scrapy.Spider):
    name = 'articles'
    allowed_domains = ['artofmanliness.com']
    max_pages = 200

    def start_requests(self):
        # 页码从1开始，替换为https协议
        for i in range(1, self.max_pages + 1):
            yield scrapy.Request(f'https://www.artofmanliness.com/articles/page/{i}/', callback=self.parse)

    def parse(self, response):
        for article in response.xpath('//article[contains(@class, "aom-article-simple")]'):
            url = article.xpath('.//a/@href').get()
            if url:
                yield Request(url=url, callback=self.parse_article)

    def parse_article(self, response):
        # 修正XPath规则适配站点当前结构
        title = response.xpath('//h1[contains(@class, "post-title")]/text()').get()
        category = response.xpath('//p[contains(@class, "category")]/a/text()').get()
        date = response.xpath('//p[contains(@class, "post-date")]/span[@class="date"]/text()').get()

        yield {
            'Title': title.strip() if title else '',
            'Category': category.strip() if category else '',
            'Date': date.strip() if date else '',
            'URL': response.url           
        }

配置说明 #

原有settings.py配置无需调整，可直接正常使用。 #

内容的提问来源于stack exchange，提问作者Ken Ruto