You need to enable JavaScript to run this app.
优惠活动
大模型
产品
解决方案
定价
更多

使用scrapy-playwright下载文件失败的问题排查求助

avatar
架构老林
2026-7-8
问题：Scrapy-Playwright下载PDF文件失败，启动后中断

尝试用scrapy-playwright下载目标PDF文件，从日志可见下载已启动但随即中断，错误为net::ERR_ABORTED，原因不明确，推测爬虫未等待下载完成。

爬虫代码 

import scrapy
from scrapy_playwright.page import PageMethod
from playwright.async_api import Dialog
import logging

class AwesomeSpider(scrapy.Spider):
    name = "awesome"

    def start_requests(self):
        # GET request
        yield scrapy.Request(
            url="https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf",
            meta=dict(
                playwright=True,
                playwright_include_page = True, 
                playwright_page_event_handlers = {
                    "download": self.handle_download
                },
                playwright_page_methods={
                    "expect_download": PageMethod("expect_download", predicate=self.is_download_done, timeout=120000),
                },
            ),
            errback=self.errback,
        )
    def is_download_done(self, download):
        logging.info(f"Is download done: {download.is_done()}")
        return download.is_done()

    async def handle_download(self, download) -> None:
        print("File download started")
        print(f"File: {download.suggested_filename}")
        await download.save_as("~/projects/nenad/temp/" + download.suggested_filename)
        print(f"Received file with path {await download.path()}")

    async def parse(self, response):
        # 'response' contains the page as seen by the browser
        logging.info("Parsing response")
        page = response.meta["playwright_page"]
        file = await page.expect_download(timeout=120000)
        # screenshot contains the image's bytes
        await page.close()
        return {"file": file}
    
    async def errback(self, failure):
        logging.info(
            "Handling failure in errback, request=%r, exception=%r", failure.request, failure.value
        )
        logging.info(failure)
        page = failure.request.meta["playwright_page"]
        
        await page.close()
        await page.context.close()

错误日志 

2023-10-16 20:37:01 [scrapy-playwright] INFO: Launching browser chromium
2023-10-16 20:37:01 [scrapy-playwright] INFO: Browser chromium launched
2023-10-16 20:37:01 [scrapy-playwright] DEBUG: Browser context started: 'default' (persistent=False, remote=False)
2023-10-16 20:37:02 [scrapy-playwright] DEBUG: [Context=default] New page created, page count is 1 (1 for all contexts)
2023-10-16 20:37:02 [scrapy-playwright] DEBUG: [Context=default] Request: <GET https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf> (resource type: document)
2023-10-16 20:37:02 [scrapy-playwright] DEBUG: [Context=default] Response: <200 https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf>
File download started
File: cenovnik-fiat-500.pdf
2023-10-16 20:37:02 [root] INFO: Handling failure in errback, request=<GET https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf>, exception=Error('net::ERR_ABORTED at https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf\n=========================== logs ===========================\nnavigating to "https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf", waiting until "load"\n============================================================')
2023-10-16 20:37:02 [root] INFO: [Failure instance: Traceback: <class 'playwright._impl._api_types.Error'>: net::ERR_ABORTED at https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf
=========================== logs ===========================
navigating to "https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf", waiting until "load"
============================================================
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:735:errback
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:798:_startRunCallbacks
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:892:_runCallbacks
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:1792:gotResult
--- <exception caught here> ---
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:1693:_inlineCallbacks
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/python/failure.py:518:throwExceptionIntoGenerator
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/scrapy/core/downloader/middleware.py:54:process_request
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/twisted/internet/defer.py:1065:adapt
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/scrapy_playwright/handler.py:322:_download_request
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/scrapy_playwright/handler.py:357:_download_request_with_page
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/async_api/_generated.py:9251:goto
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/_impl/_page.py:473:goto
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/_impl/_frame.py:138:goto
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/_impl/_connection.py:61:send
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/_impl/_connection.py:482:wrap_api_call
/home/mrav/.local/share/virtualenvs/scrapy_google-izqaQN_l/lib/python3.8/site-packages/playwright/_impl/_connection.py:97:inner_send
]
Received file with path /tmp/playwright-artifacts-Of01oV/40467222-8f9f-4c24-9e67-348d60a34a68
2023-10-16 20:37:02 [scrapy-playwright] DEBUG: Browser context closed: 'default' (persistent=False, remote=False)
2023-10-16 20:37:03 [scrapy.core.engine] INFO: Closing spider (finished)

Settings配置 

TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
DOWNLOAD_HANDLERS = {
    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}

PLAYWRIGHT_BROWSER_TYPE = "chromium"

问题原因分析

  1. 页面加载逻辑冲突：Playwright默认调用goto时等待页面load事件完成，但请求PDF文件时，浏览器会直接触发下载而非渲染HTML页面，导致导航被中止，抛出net::ERR_ABORTED错误，Scrapy请求流程进入errback。
  2. 重复监听下载事件：代码同时通过playwright_page_event_handlers注册download事件，又在parse方法中调用page.expect_download，两种监听方式可能导致逻辑冲突，干扰下载流程。
  3. 路径解析问题：save_as方法中使用~/路径，Playwright可能无法正确解析用户主目录，导致文件保存失败。
  4. 提前关闭上下文：errback中直接关闭page和context，可能中断正在进行的下载过程。

解决建议及修正代码

修正要点

  • 调整Playwright页面加载等待策略，避免因PDF下载触发导航中止错误；
  • 移除重复的下载监听逻辑，保留一种处理方式；
  • 正确解析文件保存路径；
  • 优化errback逻辑，避免提前关闭资源。

修正后的代码 

import scrapy
import os
from scrapy_playwright.page import PageMethod
import logging

class AwesomeSpider(scrapy.Spider):
    name = "awesome"

    def start_requests(self):
        yield scrapy.Request(
            url="https://www.fiat.rs/content/dam/fiat/rs/cenovnici-i-katalozi/cenovnik-fiat-500.pdf",
            meta=dict(
                playwright=True,
                playwright_include_page=True,
                playwright_page_event_handlers={
                    "download": self.handle_download
                },
                # 修改等待策略，避免因PDF下载导致导航中止报错
                playwright_page_goto_kwargs={
                    "wait_until": "domcontentloaded",
                },
            ),
            errback=self.errback,
        )

    async def handle_download(self, download) -> None:
        print("File download started")
        print(f"File: {download.suggested_filename}")
        # 解析用户主目录路径，确保路径有效
        save_dir = os.path.expanduser("~/projects/nenad/temp/")
        # 确保目录存在
        os.makedirs(save_dir, exist_ok=True)
        save_path = os.path.join(save_dir, download.suggested_filename)
        await download.save_as(save_path)
        print(f"File successfully saved to: {save_path}")
        print(f"Temporary download path: {await download.path()}")

    async def parse(self, response):
        logging.info("Parsing response")
        page = response.meta["playwright_page"]
        # 移除重复的expect_download，通过事件监听处理下载即可
        await page.close()

    async def errback(self, failure):
        logging.info(
            "Handling failure in errback, request=%r, exception=%r", failure.request, failure.value
        )
        logging.info(failure)
        page = failure.request.meta.get("playwright_page")
        if page:
            # 等待下载完成后再关闭页面
            await page.close()
        context = failure.request.meta.get("playwright_context")
        if context:
            await context.close()

内容的提问来源于stack exchange，提问作者Ivan Vasiljevic

相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分)， 限时 9.9 元起

产品介绍相关文档
相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分)， 限时 9.9 元起

产品介绍相关文档
火山引擎 最新活动
全面开放
Doubao-Seedance-2.5全新模型升级

模型再升级,30秒超长叙事, 模态参考扩容

HOT
方舟Codingplan

模型自由，工具不限，最新支持 Deepseek-V4 系列、GLM-5.3 系列

HOT
方舟Agent plan

超全模态模型 × Harness 升级，最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列

HOT
新用户特惠专场

大模型19元起，Al应用9.9元畅享，新人首购爆款尽享优惠

最近更新时间：2026.07.08 10:37:02