如何优化Scrapy中的Python Requests以提升Saks网站爬取速度？

我正在爬取Saks Fifth Avenue女装页面（https://www.saksfifthavenue.com/c/women-s-apparel），每个产品包含尺寸、颜色变体及对应库存状态，但这类信息未集中展示，必须发送带prod_id、color、size的自定义请求才能获取。这导致Scrapy爬取速度大幅下降，目前耗时超5小时仅爬取3000个产品，无法满足6小时内完成的需求，寻求优化Scrapy中Python Requests使用的方法。

以下是当前爬虫代码：

import scrapy
from urllib.parse import urlencode
import requests
from scrapy.selector import Selector
import re
import json
import html


class SaksFifthAvenueSpider(scrapy.Spider):
    name = "safa-feeds"

    # custom settings
    custom_settings = {
        "LOG_FILE": "saks_fifth_avenue.log",
        "ITEM_PIPELINES": {
            "sdt.pipelines.SdtWasabiS3Pipeline": 300,
        },
    }

    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "cross-site",
        "Sec-GPC": "1",
    }

    params = {
        "cgid": "",
        "start": "0",
        "sz": "24",
        "hideLess": "true",
    }

    base_url = "https://www.saksfifthavenue.com/on/demandware.store/Sites-SaksFifthAvenue-Site/en_US/Search-UpdateGrid?"

    def start_requests(self):
        cgid_list = [
            "2534374306418048",
            "2534374306624247",
            "2534374306622828",
            "1608313652004",
            "2534374306418050",
            "2534374306418162",
            "2534374306418054",
            "1654108780232",
            "2534374306418205",
            "2534374306418206",
            "2534374306418217",
            "2534374306418192",
            "1608313652004",
            "2534374306418053",
        ]
        for cgid in cgid_list:
            self.params["cgid"] = cgid
            category_url = self.base_url + urlencode(self.params)
            yield scrapy.Request(
                url=category_url, headers=self.headers, callback=self.parse_page_items
            )

    def parse_page_items(self, response):
        item_links = set(
            [
                "https://www.saksfifthavenue.com" + u.split("?")[0]
                for u in response.css("a.thumb-link.mw-100::attr(href)").extract()
            ]
        )
        inner_load = response.css("div.show-more ::attr(data-url)").get()
        if inner_load:
            yield scrapy.Request(
                url=inner_load, headers=self.headers, callback=self.parse_page_items
            )
        # next_page_no = response.css('a[aria-label="Next"]::attr(href)').get()
        # if next_page_no:
        #     self.params["start"] = next_page_no.split("&")[0].split("=")[-1]
        #     next_url = self.base_url + urlencode(self.params)
        #     yield scrapy.Request(
        #         url=next_url, headers=self.headers, callback=self.parse_page_items
        #     )

        for link in item_links:
            yield scrapy.Request(
                url=link, headers=self.headers, callback=self.parse_product_details
            )

    def parse_product_details(self, response):
        item = {}
        json_text = (
            response.css('script[type="application/ld+json"]')
            .get()
            .replace('<script type="application/ld+json">', "")
            .replace("</script>", "")
        )
        json_blob = json.loads(json_text)
        prod_id = response.css("div.container.product-detail::attr(data-pid)").get()
        colors = response.css("button::attr(data-adobelaunchproductcolor)").extract()
        sizes = response.css("li::attr(data-attr-value)").extract()
        item["product_id"] = prod_id
        item["product_brand"] = response.css("a.product-brand::text").get()
        item["product_name"] = response.css("h1.product-name::text").get()
        json_breadcrumbs_text = (
            response.css('script[type="application/ld+json"]')
            .extract()[-1]
            .replace('<script type="application/ld+json">', "")
            .replace("</script>", "")
        )
        bc_json_blob = json.loads(json_breadcrumbs_text)
        item["categories"] = [
            {f"category_{idx}": cat["name"]}
            for idx, cat in enumerate(bc_json_blob["itemListElement"], 1)
        ]
        item["slug"] = json_blob["offers"]["url"].split(".com")[-1]
        desc = json_blob["description"]
        item["description"] = re.sub("<[^<]+?>", " ", html.unescape(desc))
        item["product_variants"] = []
        item["color"] = response.css(
            "span.text2.color-value.attribute-displayValue::text"
        ).get()
        item["sizes"] = []
        for color in colors:
            for i_size in sizes:
                variant_url = (
                    response.url
                    + "?dwvar_"
                    + prod_id
                    + "_color="
                    + color.upper()
                    + "&dwvar_"
                    + prod_id
                    + f"_size={i_size}&pid="
                    + prod_id
                )
                resp = requests.get(variant_url, headers=self.headers)
                product_variants = Selector(text=resp.text)
                size = "".join(
                    list(
                        filter(
                            None,
                            [
                                s.replace("\n", "")
                                for s in product_variants.css("li")
                                .css("[selected] ::text")
                                .extract()
                            ],
                        )
                    )
                )
                disabled = (
                    product_variants.css("li")
                    .css("[disabled]")
                    .css("[selected] ::text")
                    .getall()
                )
                final_price = ""
                final_price = product_variants.css(
                    "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-list-price::text"
                ).get()
                if final_price is None:
                    final_price = product_variants.css(
                        "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-sale-price::text"
                    ).get()
                try:
                    old_price = product_variants.css(
                        "span.formatted_price.bfx-price.bfx-list-price ::text"
                    ).get()
                except:
                    old_price = ""

                if not disabled:
                    item["product_variants"].append(
                        {
                            "color": product_variants.css(
                                "span.text2.color-value.attribute-displayValue::text"
                            ).get(),
                            "size": size,
                            "status": "AVAILABLE",
                            "final_price": final_price,
                            "old_price": old_price,
                        }
                    )
                else:
                    item["product_variants"].append(
                        {
                            "color": product_variants.css(
                                "span.text2.color-value.attribute-displayValue::text"
                            ).get(),
                            "size": size,
                            "status": "NOT_AVAILABLE",
                            "final_price": final_price,
                            "old_price": old_price,
                        }
                    )

        if item["product_variants"] == []:
            size_selector = response.css(
                "ul.radio-group-list.size-attribute.swatch-display-three.show-size-dropdown"
            )
            for s in size_selector.css("li"):
                all_size_var = s.css("::text").getall()
                if not s.css("[disabled]"):
                    available = all_size_var
                    clean = list(filter(None, [c.replace("\n", "") for c in available]))
                    for out_si in clean:
                        item["sizes"].append({"size": out_si, "status": "AVAILABLE"})
                else:
                    out_of_stock = all_size_var
                    clean = list(
                        filter(None, [c.replace("\n", "") for c in out_of_stock])
                    )
                    for in_si in clean:
                        item["sizes"].append({"size": in_si, "status": "NOT_AVAILABLE"})

        if item["product_variants"] == [] and item["sizes"] == []:
            if response.css("div.form-group.show-size-dropdown-holder"):
                size_dropdown = response.css(
                    "ul.radio-group-list.size-attribute.swatch-display-three ::text"
                ).extract()
                clean_sizes = list(
                    filter(None, [s.replace("\n", "") for s in size_dropdown])
                )

                for dd_si in clean_sizes:
                    variant_url = (
                        response.url
                        + "?dwvar_"
                        + prod_id
                        + "_color="
                        + item["color"].upper()
                        + "&dwvar_"
                        + prod_id
                        + f"_size={dd_si}&pid="
                        + prod_id
                    )
                    resp = requests.get(variant_url, headers=self.headers)
                    product_variants = Selector(text=resp.text)
                    size = "".join(
                        list(
                            filter(
                                None,
                                [
                                    s.replace("\n", "")
                                    for s in product_variants.css("li")
                                    .css("[selected] ::text")
                                    .extract()
                                ],
                            )
                        )
                    )
                    disabled = (
                        product_variants.css("li")
                        .css("[disabled]")
                        .css("[selected] ::text")
                        .getall()
                    )
                    final_price = ""
                    final_price = product_variants.css(
                        "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-list-price::text"
                    ).get()
                    if final_price is None:
                        final_price = product_variants.css(
                            "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-sale-price::text"
                        ).get()
                    try:
                        old_price = product_variants.css(
                            "span.formatted_price.bfx-price.bfx-list-price ::text"
                        ).get()
                    except:
                        old_price = ""

                    if not disabled:
                        item["product_variants"].append(
                            {
                                "color": item["color"],
                                "size": size,
                                "status": "AVAILABLE",
                                "final_price": final_price,
                                "old_price": old_price,
                            }
                        )
                    else:
                        item["product_variants"].append(
                            {
                                "color": item["color"],
                                "size": size,
                                "status": "NOT_AVAILABLE",
                                "final_price": final_price,
                                "old_price": old_price,
                            }
                        )

        item["gender"] = ""
        bc_li = [b["name"] for b in bc_json_blob["itemListElement"]]

        if "Women's Clothing" in bc_li:
            item["gender"] = "Female"
        elif "Men" in bc_li or "Men's" in bc_li:
            item["gender"] = "Male"
        else:
            item["gender"] = "Female"
        if (
            "Kids" in bc_li
            and any("Boys" in s for s in bc_li)
            or any("Baby Boy" in s for s in bc_li)
        ):
            item["gender"] = "Boy"
        elif (
            "Kids" in bc_li
            and any("Girls" in s for s in bc_li)
            or any("Baby Girl" in s for s in bc_li)
        ):
            item["gender"] = "Girl"

        elif (
            any("Kids" in s for s in bc_li)
            and not any("Baby Girl" in s for s in bc_li)
            and not any("Baby Boy" in s for s in bc_li)
            and not any("Boys" in s for s in bc_li)
            and not any("Girls" in s for s in bc_li)
        ):
            item["gender"] = "Kids"

        elif any("Accessories" in s for s in bc_li):
            item["gender"] = ""

        else:
            item["gender"] = ""

        price_json_text = (
            response.css('script[type="text/javascript"]')
            .extract()[2]
            .replace('<script type="text/javascript">\npageDataObj = ', "")
            .replace(";\n</script>", "")
        )
        price_json_blob = json.loads(price_json_text)
        item["tag"] = price_json_blob["products"][0]["tags"]["feature_type"]
        item["price"] = [
            {
                "original_price": p["original_price"],
                "price": p["price"],
                "currency": json_blob["offers"]["priceCurrency"],
            }
            for p in price_json_blob["products"]
        ]
        item["images"] = json_blob["image"]

        yield item

优化建议 #

1. 替换同步requests.get为Scrapy异步请求 #

这是速度慢的核心原因：requests.get()是阻塞式同步请求，会让Scrapy的异步调度完全失效，每个变体请求必须等前一个完成才能继续。

修改方案：
将变体请求改为scrapy.Request，通过meta参数传递上下文数据（如当前item、颜色、尺寸），新增回调函数处理变体响应：

# 替换parse_product_details里的嵌套循环
for color in colors:
    for i_size in sizes:
        variant_url = f"{response.url}?dwvar_{prod_id}_color={color.upper()}&dwvar_{prod_id}_size={i_size}&pid={prod_id}"
        yield scrapy.Request(
            url=variant_url,
            headers=self.headers,
            callback=self.parse_variant,
            meta={
                'item': item.copy(),  # 传递item副本，避免被后续请求修改
                'color': color,
                'size': i_size,
                'total_variants': len(colors)*len(sizes),
                'current_count': 0
            }
        )

新增parse_variant函数处理变体响应：

def parse_variant(self, response):
    item = response.meta['item']
    color = response.meta['color']
    i_size = response.meta['size']
    
    # 直接用response，无需再转Selector
    size = "".join(
        list(filter(None, [s.replace("\n", "") for s in response.css("li[selected] ::text").extract()]))
    )
    disabled = response.css("li[disabled][selected] ::text").getall()
    final_price = response.css(
        "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-list-price::text"
    ).get() or response.css(
        "span.formatted_sale_price.formatted_price.js-final-sale-price.bfx-price.bfx-sale-price::text"
    ).get() or ""
    old_price = response.css("span.formatted_price.bfx-price.bfx-list-price ::text").get() or ""

    variant = {
        "color": response.css("span.text2.color-value.attribute-displayValue::text").get(),
        "size": size,
        "status": "AVAILABLE" if not disabled else "NOT_AVAILABLE",
        "final_price": final_price,
        "old_price": old_price
    }
    item["product_variants"].append(variant)
    
    # 统计请求完成数，全部完成后再yield item
    item['_variant_count'] = item.get