如何用Scrapy处理IMDB加载更多按钮以爬取全部影评？

我需要爬取IMDB电影数据，目前使用Scrapy框架已能提取电影标题等基础信息，但在爬取影评时遇到了加载更多按钮的问题。我的起始URL为：https://www.imdb.com/search/title/?title_type=feature&primary_language=hi&sort=num_votes,desc

附上现有代码：

def parse(self, response):

    for result in response.css('.mode-advanced'):

        primaryTitle = result.css('div.lister-item-content h3.lister-item-header a::text').extract_first()          
        link = result.css('div.lister-item-content h3.lister-item-header a::attr(href)').extract_first()
        duration = result.css('div.lister-item-content p:nth-child(2) span.runtime::text').extract_first()
        genre = result.css('div.lister-item-content p:nth-child(2) span.genre::text').extract_first()

        yield scrapy.Request(url='https://www.imdb.com'+link,
                                callback=self.parse_listing,
                                meta={'primaryTitle':primaryTitle,
                                    'duration':duration,
                                    'genre':genre})
        

    next_page = response.css('.lister-page-next::attr(href)').get()
    if next_page is not None:
        yield scrapy.Request(url='https://www.imdb.com'+next_page, callback=self.parse)


def parse_listing(self, response):
    primaryTitle = response.meta['primaryTitle']
    duration = response.meta['duration']
    genre = response.meta['genre']

    fullTitle = response.css('.fbQftq h1.eKrKux::text').extract_first()
    description = response.css('div.hrgVKw span::text').extract()
    year = response.css('.fbQftq div.iJtmbR ul.baseAlt li:nth-child(1) a.WIUyh::text').extract_first()
    rating = response.css('.fAePGh div.kYEdvH span.jGRxWM::text').extract_first()
    director = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(1) div ul.baseAlt li a::text').extract()
    writer = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(2) div ul.baseAlt li a::text').extract()
    stars = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(3) div ul.baseAlt li a::text').extract()
    votes = response.css('div.bZeUlh div.gopMqI::text').extract_first()

    review_link = response.css('div.gcCaSg ul.baseAlt li.ghlYSH a.isReview::attr(href)').extract_first()

    yield scrapy.Request(url='https://www.imdb.com'+review_link,
                        callback=self.parse_review,
                        meta={'primaryTitle':primaryTitle,
                                'duration':duration,
                                'genre':genre,
                                'fullTitle':fullTitle,
                                'description':description,
                                'year':year,
                                'rating':rating,
                                'director':director,
                                'writer': writer,
                                'stars':stars,
                                'votes':votes})

def parse_review(self, response):
    primaryTitle = response.meta['primaryTitle']
    duration = response.meta['duration']
    genre = response.meta['genre']
    fullTitle = response.meta['fullTitle']
    description = response.meta['description']
    year = response.meta['year']
    rating = response.meta['rating']
    director = response.meta['director']
    writer = response.meta['writer']
    stars = response.meta['stars']
    votes = response.meta['votes']

    for review_details in response.css('div.lister-item-content'):

        review_rating = review_details.css('div.ipl-ratings-bar span.rating-other-user-rating span::text').extract_first()
        review_title = review_details.css('a.title::text').extract_first()
        username = review_details.css('div.display-name-date span.display-name-link a::text').extract_first()
        review_date = review_details.css('div.display-name-date span.review-date::text').extract_first()
        review_description = review_details.css('div.content div.show-more__control::text').extract_first()

        yield{'url': response.url,
            'primaryTitle': primaryTitle,
            'duration': duration,
            'genre': genre,
            'fullTitle': fullTitle,
            'description': description,
            'year': year,
            'duration':duration,
            'rating':rating,
            'director':director,
            'rating':rating,
            'director':director,
            'writer':writer,
            'stars':stars,
            'votes':votes,
            'review_rating':review_rating,
            'review_title':review_title,
            'username':username,
            'review_date':review_date,
            'description':description}

    
    base_url = 'https://www.imdb.com'
    review_ajaxurl = '/title/tt10811166/reviews/_ajax'
    paginationKey = 'g4wp7bjfqy2tg3yl7swh7nzqrht4uabhzfmxvlnomwklyczuf43o6ss4pe2vlmjncr4k4wendyfpwpblleaqtzauqo4rtea'


    load_more_url = base_url+review_ajaxurl+'?ref_=undefined&paginationKey='+paginationKey
    if load_more_url is not None:
        yield scrapy.Request(url = load_more_url, callback=self.parse_review)

请问如何通过Scrapy爬取IMDB电影的全部影评？

你的核心问题在于硬编码了固定的电影ID和分页密钥，导致无法动态处理不同电影的影评加载，也无法持续获取下一页影评。以下是修改后的完整方案：

1. 动态提取影评AJAX接口地址和分页密钥 #

IMDB的影评加载更多是通过AJAX请求实现的，每一页的分页密钥（paginationKey）会在当前页面的HTML中返回，你需要从影评页面的load-more-data属性里提取。

2. 传递元数据到AJAX请求回调 #

因为AJAX返回的影评页面只包含影评内容，没有电影的基础信息，所以需要通过meta参数把之前获取的电影元数据一直传递下去。

修改后的完整代码 #

def parse(self, response):
    for result in response.css('.mode-advanced'):
        primaryTitle = result.css('div.lister-item-content h3.lister-item-header a::text').extract_first()          
        link = result.css('div.lister-item-content h3.lister-item-header a::attr(href)').extract_first()
        duration = result.css('div.lister-item-content p:nth-child(2) span.runtime::text').extract_first()
        genre = result.css('div.lister-item-content p:nth-child(2) span.genre::text').extract_first()

        yield scrapy.Request(
            url='https://www.imdb.com' + link,
            callback=self.parse_listing,
            meta={
                'primaryTitle': primaryTitle,
                'duration': duration,
                'genre': genre
            }
        )
    
    next_page = response.css('.lister-page-next::attr(href)').get()
    if next_page is not None:
        yield scrapy.Request(url='https://www.imdb.com' + next_page, callback=self.parse)


def parse_listing(self, response):
    meta = response.meta.copy()
    meta['fullTitle'] = response.css('.fbQftq h1.eKrKux::text').extract_first()
    meta['description'] = response.css('div.hrgVKw span::text').extract()
    meta['year'] = response.css('.fbQftq div.iJtmbR ul.baseAlt li:nth-child(1) a.WIUyh::text').extract_first()
    meta['rating'] = response.css('.fAePGh div.kYEdvH span.jGRxWM::text').extract_first()
    meta['director'] = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(1) div ul.baseAlt li a::text').extract()
    meta['writer'] = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(2) div ul.baseAlt li a::text').extract()
    meta['stars'] = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(3) div ul.baseAlt li a::text').extract()
    meta['votes'] = response.css('div.bZeUlh div.gopMqI::text').extract_first()

    review_link = response.css('div.gcCaSg ul.baseAlt li.ghlYSH a.isReview::attr(href)').extract_first()
    if review_link:
        yield scrapy.Request(
            url='https://www.imdb.com' + review_link,
            callback=self.parse_review,
            meta=meta
        )


def parse_review(self, response):
    meta = response.meta.copy()
    
    # 提取当前页的影评
    for review_details in response.css('div.lister-item-content'):
        review_rating = review_details.css('div.ipl-ratings-bar span.rating-other-user-rating span::text').extract_first()
        review_title = review_details.css('a.title::text').extract_first()
        username = review_details.css('div.display-name-date span.display-name-link a::text').extract_first()
        review_date = review_details.css('div.display-name-date span.review-date::text').extract_first()
        review_description = review_details.css('div.content div.show-more__control::text').extract_first()

        yield {
            'url': response.url,
            'primaryTitle': meta['primaryTitle'],
            'duration': meta['duration'],
            'genre': meta['genre'],
            'fullTitle': meta['fullTitle'],
            'description': meta['description'],
            'year': meta['year'],
            'rating': meta['rating'],
            'director': meta['director'],
            'writer': meta['writer'],
            'stars': meta['stars'],
            'votes': meta['votes'],
            'review_rating': review_rating,
            'review_title': review_title,
            'username': username,
            'review_date': review_date,
            'review_description': review_description  # 修正重复的description键
        }
    
    # 动态获取加载更多的AJAX链接和分页密钥
    load_more_element = response.css('div.load-more-data')
    if load_more_element:
        ajax_url = load_more_element.attrib.get('data-ajaxurl')
        pagination_key = load_more_element.attrib.get('data-key')
        if ajax_url and pagination_key:
            load_more_url = f'https://www.imdb.com{ajax_url}?ref_=undefined&paginationKey={pagination_key}'
            yield scrapy.Request(
                url=load_more_url,
                callback=self.parse_review,
                meta=meta  # 传递电影元数据到下一页
            )

关键修改点说明 #

• 动态提取AJAX信息：从影评页面的div.load-more-data元素中获取data-ajaxurl（AJAX接口地址）和data-key（分页密钥），替代原来的硬编码值。
• 元数据传递优化：使用meta.copy()复制元数据，避免修改原字典导致的问题，确保每一页影评都能关联到对应的电影信息。
• 修正数据键重复：原代码中yield字典里重复了duration、rating、director、description键，已修正为review_description区分影评内容和电影简介。
• 容错处理：增加了对review_link、ajax_url、pagination_key的存在性判断，避免空值导致的请求错误。

这样修改后，爬虫就能自动遍历每一部电影的所有影评页面，直到没有更多影评可加载为止。

内容的提问来源于stack exchange，提问作者riya23