使用Scrapy从REST API爬取多分类数据：子分类提取及标签处理难题

问题1：子分类层级无法获取 #

你用XPath解析JSON响应的方式完全错误——当前请求返回的是纯JSON数据，XPath仅适用于HTML/XML文档，自然拿不到任何内容。要获取产品的完整分类层级，需要修改GraphQL查询语句，让API返回每个产品的分类关联信息。

解决步骤： #

1. 修改GraphQL Query：在products.items的字段列表中添加categories字段，获取每个产品所属的所有分类（包含父级、子级分类的名称和路径）。
2. 提取并拼接层级：从返回的分类数据中筛选出完整的层级链，拼接成主分类>子分类>子子分类的格式。

问题2：产品描述含HTML标签 #

直接提取short_description.html会保留<p>等HTML标签，需要清理为纯文本。

解决步骤： #

使用w3lib.html.remove_tags()方法移除所有HTML标签，或者用Scrapy内置的strip_tags工具。

修改后的完整代码 #

import scrapy
from scrapy.crawler import CrawlerProcess
from datetime import datetime
from w3lib.html import remove_tags  # 导入清理HTML标签的工具

categories = {
    "6011": {"pages": 60, "name": "Цахилгаан бараа"},
    "24175": {"pages": 70, "name": "Хүнс"},
    "24273": {"pages": 40, "name": "Гэр ахуй"},
    "21297": {"pages": 70, "name": "Гоо сайхан"},
    "19653": {"pages": 30, "name": "Гутал, хувцас"},
    "19451": {"pages": 10, "name": "Авто бараа"},
    "19518": {"pages": 40, "name": "Барилгын материал"},
    "19853": {"pages": 10, "name": "Аялал, Спорт бараа"},
    "19487": {"pages": 50, "name": "Ном"},
    "19767": {"pages": 20, "name": "Бичиг хэрэг"},
    "19469": {"pages": 10, "name": "Эрүүл мэнд"},
    "19545": {"pages": 20, "name": "Хүүхдийн бараа"},
}

dt_today = datetime.now().strftime('%Y%m%d')
filename = dt_today + ' Nomin'


class Nomin(scrapy.Spider):
    name = 'nomin_cpi'
    custom_settings = {
        "FEEDS": {
            f'{filename}.csv': {
                'format': 'csv',
                'overwrite': True}}
    }

    def start_requests(self):
        for cat, val in categories.items():
            for page in range(1, val["pages"]):
                # 修改GraphQL Query，添加categories字段
                url = f'https://eshop.nomin.mn/graphql?query=query+category%28%24pageSize%3AInt%21%24currentPage%3AInt%21%24filters%3AProductAttributeFilterInput%21%24sort%3AProductAttributeSortInput%29%7Bproducts%28pageSize%3A%24pageSize+currentPage%3A%24currentPage+filter%3A%24filters+sort%3A%24sort%29%7Bitems%7Bid+name+sku+brand+salable_qty+brand_name+c21_available+c21_business_type+c21_reference+c21_street+c21_area+c21_bed_room+mp_daily_deal%7Bcreated_at+date_from+date_to+deal_id+deal_price+remaining_time+deal_qty+discount_label+is_featured+product_id+product_name+product_sku+sale_qty+status+store_ids+updated_at+__typename%7Dnew_to_date+short_description%7Bhtml+__typename%7DproductAttributes%7Bname+value+__typename%7Dprice%7BregularPrice%7Bamount%7Bcurrency+value+__typename%7D__typename%7D__typename%7Dspecial_price+special_to_date+thumbnail%7Bfile_small+url+__typename%7Durl_key+url_suffix+mp_label_data%7Benabled+name+priority+label_template+label_image+to_date+__typename%7Dcategories%7Bname+path+__typename%7D...on+ConfigurableProduct%7Bvariants%7Bproduct%7Bsku+special_price+price%7BregularPrice%7Bamount%7Bcurrency+value+__typename%7D__typename%7D__typename%7D__typename%7D__typename%7D__typename%7D__typename%7Dpage_info%7Btotal_pages+__typename%7Dtotal_count+__typename%7D%7D&operationName=category&variables=%7B%22currentPage%22%3A{page}%2C%22id%22%3A{cat}%2C%22filters%22%3A%7B%22category_id%22%3A%7B%22in%22%3A%22{cat}%22%7D%7D%2C%22pageSize%22%3A50%2C%22sort%22%3A%7B%22news_from_date%22%3A%22ASC%22%7D%7D'
                yield scrapy.Request(
                    url,
                    callback=self.parse,
                    cb_kwargs={"category": val["name"]}
                )

    def parse(self, response, category=None):
        data = response.json()
        if data and data['data'] and data['data']['products'] and data['data']['products']['items']:
            for item in data['data']["products"]["items"]:
                # 提取分类层级：从categories中筛选出完整路径，拼接成>分隔的字符串
                category_hierarchy = []
                if item.get('categories'):
                    # 按path长度排序，确保从主分类到子分类的顺序
                    sorted_cats = sorted(item['categories'], key=lambda x: len(x['path']))
                    category_hierarchy = [cat['name'] for cat in sorted_cats]
                full_category = '>'.join(category_hierarchy) if category_hierarchy else category

                # 清理产品描述的HTML标签
                clean_description = remove_tags(item["short_description"]["html"]) if item.get("short_description") else ""

                yield {
                    "Барааны нэр": item["name"],
                    "Category": full_category,
                    "Үндсэн үнэ": item["price"]["regularPrice"]["amount"]["value"],
                    "Хөнгөлөлтийн үнэ": item["special_price"],
                    "Барааны тайлбар": clean_description,
                    "Барааны код": item["sku"]
                }


if __name__ == "__main__":
    process = CrawlerProcess()
    process.crawl(Nomin)
    process.start()

关键修改说明： #

1. GraphQL Query更新：在items字段中添加了categories{name path __typename}，让API返回每个产品的分类名称和路径。
2. 分类层级处理：对返回的分类按路径长度排序，确保层级顺序正确，再拼接成>分隔的字符串。
3. HTML标签清理：用w3lib.html.remove_tags()移除描述中的所有HTML标签，得到纯文本内容。
4. 移除无效代码：删除了原代码中无效的Category2字段（基于XPath的错误尝试）。

内容的提问来源于stack exchange，提问作者WX1505