Scrapy爬虫如何避免JSON文件中存储重复团队数据

爬取赛车网站时，以团队列表为入口获取每个团队关联骑手的全部信息，但生成的JSON文件中存在重复的团队对象，仅最后一条包含完整的骑手数据。示例重复数据如下：

[
  {'team' : 'team-1', '450' : [{rider_obj- 1}], '250' : [{rider_obj- 1}]},
  {'team' : 'team-2', '450' : [{rider_obj- 1}], '250' : [{rider_obj- 1}]},
  {'team' : 'team-1', '450' : [{rider_obj- 1}, {rider_obj- 2}], '250' : [{rider_obj- 1}]},
  {'team' : 'team-1', '450' : [{rider_obj- 1}, {rider_obj- 2}], '250' : [{rider_obj- 1}, {rider_obj- 2}, {rider_obj- 3}]}
]

原爬虫代码如下：

class PromotocrossSpider(scrapy.Spider):
    name = "promotocross"
    allowed_domains = ["mydomain"]
    start_urls = ["mywebsite"]
    custom_settings = {
        'FEED_FORMAT': 'json',
        'FEED_URI': 'promotocross.json',
    }

    def parse(self, response):

        for section in response.css('section.component_breathe'):
            teams_data = {}
            teams_data['team'] = section.css('header h2 a::text').get().strip()
            classes = section.css('div.card_content h3::text').getall()
            dives = section.css('div.grid.grid_breathe').getall()

            for _cls, div in zip(classes, dives):
                _cls = _cls.split(" ")[0]
                teams_data[_cls] = []
                div_selector = scrapy.Selector(text=div, type="html")
                for url in div_selector.css('a::attr("href")'):
                    yield response.follow(url.get(), callback=self.parse_bio,
                                           cb_kwargs={'teams_data': teams_data, '_class': _cls})

    def parse_bio(self, response, teams_data, _class):
        rider_info = dict()
        rider_info['rider_name'] = response.css('div.rider_info h2::text').get().strip()
        rider_info['accomplishments'] = [item.get().strip() for item in response.css('div.s_100.m_50.text_content ul li::text')]
        rider_info['dob'] = response.css('div.s_100.m_50.text_content p::text')[0].get().strip()
        rider_info['city'] = response.css('div.rider_location::text').get().strip().split(",")[0]

        url = response.css('nav.nav.nav_horiz.nav_secondary ul li a')[2].attrib.get('href')
        points_stanging_url = f"https://promotocross.com{url}"
        yield response.follow(points_stanging_url, callback=self.parse_points, cb_kwargs={'rider_info': rider_info, "teams_data":teams_data, "_class":_class})


    def parse_points(self, response,  rider_info, teams_data, _class):

        points_standing_item = {}
        points_standing_item['season'] = []
        points_standing_item['Class'] = []
        points_standing_item['Overall'] = []
        points_standing_item['Points'] = []

        for tr in response.css('tbody tr'):
            tds = tr.css('td')
            points_standing_item['season'].append(tds[0].css('a::text').get().strip())
            points_standing_item['Class'].append(f"{tds[1].css('span.hidden ::text').get().strip()}")
            tds = tr.css('td::text')
            points_standing_item['Overall'].append(tds[1].get().strip())
            points_standing_item['Points'].append(tds[2].get().strip())

        rider_info['points_standing'] = points_standing_item

        url = response.css('nav.nav.nav_horiz.nav_secondary ul li a')[3].attrib.get('href')
        result_url = f"https://promotocross.com{url}"
        yield response.follow(result_url, callback=self.parse_results, cb_kwargs={'rider_info': rider_info, "teams_data":teams_data, "_class":_class})


    def parse_results(self, response,  rider_info, teams_data, _class):

        resuls_item = {}
        resuls_item['Position'] = []
        resuls_item['Race'] = []
        resuls_item['Class'] = []
        resuls_item['Date'] = []
        resuls_item['Bike'] = []

        for tr in response.css('tbody tr'):
            resuls_item['Position'].append(tr.css('td::text')[0].get().strip())
            resuls_item['Race'].append(f"{tr.css('td')[1].css('a span::text').get().strip()}")
            # resuls_item['Date'].append(tr.css('td.sorter-simpleDate::text').get)

        rider_info['results_data'] = resuls_item
        teams_data[_class].append(rider_info)

        yield teams_data

原代码中，每处理完一个骑手的所有数据后，就会立即yield团队对象。由于所有骑手共享同一个teams_data对象，每次添加新骑手后都会输出一次当前状态的团队数据，导致JSON中出现同一团队的多个条目，只有最后一次输出才包含完整的骑手信息。

修改代码，等待当前团队的所有骑手数据都处理完成后，再一次性输出完整的团队对象。具体步骤如下：

1. 导入Twisted的DeferredList，用于等待多个异步请求完成
2. 在parse方法中，先初始化团队数据，收集当前团队所有骑手的请求
3. 使用DeferredList等待所有骑手请求处理完毕后，再输出完整的团队数据
4. 移除后续解析方法中yield teams_data的语句，仅将骑手数据添加到团队对象中

修改后的完整代码：

from twisted.internet.defer import DeferredList
import scrapy

class PromotocrossSpider(scrapy.Spider):
    name = "promotocross"
    allowed_domains = ["mydomain"]
    start_urls = ["mywebsite"]
    custom_settings = {
        'FEED_FORMAT': 'json',
        'FEED_URI': 'promotocross.json',
    }

    def parse(self, response):
        for section in response.css('section.component_breathe'):
            teams_data = {}
            teams_data['team'] = section.css('header h2 a::text').get().strip()
            classes = section.css('div.card_content h3::text').getall()
            dives = section.css('div.grid.grid_breathe').getall()
            
            # 初始化每个赛事类别的骑手列表
            for _cls, div in zip(classes, dives):
                _cls = _cls.split(" ")[0]
                teams_data[_cls] = []
            
            # 收集当前团队所有骑手的请求
            rider_requests = []
            for _cls, div in zip(classes, dives):
                _cls = _cls.split(" ")[0]
                div_selector = scrapy.Selector(text=div, type="html")
                for url in div_selector.css('a::attr("href")'):
                    req = response.follow(
                        url.get(), 
                        callback=self.parse_bio,
                        cb_kwargs={'teams_data': teams_data, '_class': _cls}
                    )
                    rider_requests.append(req)
            
            # 等待所有骑手请求完成后，输出完整团队数据
            yield DeferredList(rider_requests, consumeErrors=True).addCallback(lambda _: teams_data)

    def parse_bio(self, response, teams_data, _class):
        rider_info = dict()
        rider_info['rider_name'] = response.css('div.rider_info h2::text').get().strip()
        rider_info['accomplishments'] = [item.get().strip() for item in response.css('div.s_100.m_50.text_content ul li::text')]
        rider_info['dob'] = response.css('div.s_100.m_50.text_content p::text')[0].get().strip()
        rider_info['city'] = response.css('div.rider_location::text').get().strip().split(",")[0]

        url = response.css('nav.nav.nav_horiz.nav_secondary ul li a')[2].attrib.get('href')
        points_stanging_url = f"https://promotocross.com{url}"
        yield response.follow(points_stanging_url, callback=self.parse_points, cb_kwargs={'rider_info': rider_info, "teams_data":teams_data, "_class":_class})


    def parse_points(self, response,  rider_info, teams_data, _class):
        points_standing_item = {}
        points_standing_item['season'] = []
        points_standing_item['Class'] = []
        points_standing_item['Overall'] = []
        points_standing_item['Points'] = []

        for tr in response.css('tbody tr'):
            tds = tr.css('td')
            points_standing_item['season'].append(tds[0].css('a::text').get().strip())
            points_standing_item['Class'].append(f"{tds[1].css('span.hidden ::text').get().strip()}")
            tds = tr.css('td::text')
            points_standing_item['Overall'].append(tds[1].get().strip())
            points_standing_item['Points'].append(tds[2].get().strip())

        rider_info['points_standing'] = points_standing_item

        url = response.css('nav.nav.nav_horiz.nav_secondary ul li a')[3].attrib.get('href')
        result_url = f"https://promotocross.com{url}"
        yield response.follow(result_url, callback=self.parse_results, cb_kwargs={'rider_info': rider_info, "teams_data":teams_data, "_class":_class})


    def parse_results(self, response,  rider_info, teams_data, _class):
        resuls_item = {}
        resuls_item['Position'] = []
        resuls_item['Race'] = []
        resuls_item['Class'] = []
        resuls_item['Date'] = []
        resuls_item['Bike'] = []

        for tr in response.css('tbody tr'):
            resuls_item['Position'].append(tr.css('td::text')[0].get().strip())
            resuls_item['Race'].append(f"{tr.css('td')[1].css('a span::text').get().strip()}")
            # resuls_item['Date'].append(tr.css('td.sorter-simpleDate::text').get)

        rider_info['results_data'] = resuls_item
        teams_data[_class].append(rider_info)
        # 移除原有的yield语句，不再中途输出团队数据

修改后，每个团队只会在所有骑手数据都爬取并整理完成后，才会被输出一次到JSON文件中，不会出现重复的团队对象，且每条团队数据都包含该团队下的所有骑手信息。

内容的提问来源于stack exchange，提问作者Mariyam Bloch