Scrapy+Scrapy-playwright爬Steam模组评论仅获每页首条的解决方法

问题分析 #

1. 仅获取1条评论：Steam评论采用懒加载机制，初始仅加载少量评论，且原选择器范围可能不准确，导致只匹配到单个元素。
2. 翻页逻辑冗余：原代码重复处理Cookie弹窗，且翻页后未充分等待新页面评论加载完成。

修改后的代码 #

import scrapy
import asyncio
from scrapy_playwright.page import PageMethod
from ..items import WorkshopCommentsItem


class A81tilesSpider(scrapy.Spider):
    name = '81tiles'
    start_urls = ['https://steamcommunity.com/sharedfiles/filedetails/?id=2881031511']

    def start_requests(self):
        yield scrapy.Request(
            url=self.start_urls[0],
            meta={
                "playwright": True,
                "playwright_include_page": True,
                "playwright_page_methods": [
                    PageMethod("wait_for_selector", ".commentthread_comments_list", timeout=60000)
                ],
                "page_number": 1
            }
        )

    async def parse(self, response):
        page = response.meta["playwright_page"]
        page_number = response.meta["page_number"]
        print(f"Processing page {page_number}")

        # 仅处理一次Cookie弹窗
        try:
            await page.wait_for_selector('#rejectAllButton', state='visible', timeout=10000)
            await page.click('#rejectAllButton')
            await page.wait_for_timeout(2000)
        except:
            pass

        while True:
            # 滚动加载当前页所有评论
            await self.load_all_comments(page)

            # 精准定位评论列表内的所有评论项
            all_div_comments = await page.query_selector_all('.commentthread_comments_list div.commentthread_comment_container')
            print(f"Number of comments evaluated on page {page_number}: {len(all_div_comments)}")
            
            for comment in all_div_comments:
                steam_item = WorkshopCommentsItem()
                # 容错处理：避免元素不存在导致爬虫中断
                time_elem = await comment.query_selector('.commentthread_comment_timestamp')
                steam_item['post_time'] = await time_elem.inner_text() if time_elem else 'N/A'
                
                content_elem = await comment.query_selector('.commentthread_comment_text')
                steam_item['post_content'] = await content_elem.inner_text() if content_elem else 'N/A'
                
                author_elem = await comment.query_selector('bdi')
                steam_item['post_author'] = await author_elem.inner_text() if author_elem else 'N/A'

                yield steam_item

            await asyncio.sleep(3)

            # 检查下一页按钮是否可用
            next_btn = await page.query_selector('#commentthread_PublishedFile_Public_76561198262198841_2881031511_fpagebtn_next')
            if next_btn:
                is_disabled = await page.evaluate('(btn) => btn.disabled', next_btn)
                if not is_disabled:
                    print("Clicking on Next button")
                    await next_btn.click()
                    page_number += 1
                    print(f"Processing page {page_number}")
                    
                    # 等待新页面评论加载完成
                    await page.wait_for_selector('.commentthread_comments_list', state='visible', timeout=60000)
                    await page.wait_for_timeout(3000)
                    continue
            
            # 无下一页则退出循环
            break

        await page.close()

    async def load_all_comments(self, page):
        """滚动页面触发懒加载，加载当前页所有评论"""
        last_height = await page.evaluate('document.body.scrollHeight')
        while True:
            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
            await page.wait_for_timeout(2000)
            new_height = await page.evaluate('document.body.scrollHeight')
            if new_height == last_height:
                break
            last_height = new_height

关键改动说明 #

• Cookie弹窗优化：移至循环外仅执行一次，避免重复无效操作。
• 评论加载逻辑：新增滚动加载方法，确保当前页所有懒加载评论都被触发加载。
• 选择器精准化：缩小选择范围至评论列表容器内，避免匹配页面其他位置的同类元素。
• 容错处理：为每个元素查询添加存在性判断，防止个别异常评论导致爬虫中断。
• 翻页逻辑简化：直接判断下一页按钮的禁用状态，点击后增加等待确保新页面评论加载完成。

内容的提问来源于stack exchange，提问作者cookiedough