Python爬虫无法读取JS渲染内容，求页面渲染等待解决方案

• 开发某知名电商网站爬虫，代码可运行但无法读取JS脚本，目标页面有40余个商品，仅能爬取到10个，需实现等待页面渲染后再爬取的Python请求逻辑。
• 使用requests_html的HTMLSession调用render方法时，出现错误：There is no current event loop in thread 'Thread-1 (process_request_thread)'。

import requests, random
from django.shortcuts import render
from bs4 import BeautifulSoup
from requests_html import HTMLSession, AsyncHTMLSession


# Walmart - Create your views here.
def wlista(request):

    buscarprods = request.COOKIES['buscarprod']
    url = 'https://www.walmart.com/search?q=hp+printers'  
    url = url.replace(" ", "%20")
    proveedor = request.COOKIES['proveedor'] 

    HDRS = {
       'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
       'Accept-Language': 'es-ES;es;q=0.8',
       'DNT': '1',
       'Connection': 'keep-alive',
       'Upgrade-Insecure-Requests': '1',
    }

    # This works but doesn't wait for javascript
    session_object = requests.Session()
    r = session_object.get(url, headers=HDRS).text

    # This works too, but doesn't wait for javascript
    #r = requests.get(url, headers=HDRS, timeout=(8.05, 35)).content

    #Here: I have error with "r.html.render(sleep=2)":
    # Error:
    # "There is no current event loop in thread 'Thread-1 (process_request_thread)'."

    #s  = HTMLSession()
    #r = s.get(url, headers=HDRS)
    #r.html.render(sleep=2)

    soup = BeautifulSoup(r, "html.parser")
    rows = soup.find_all(attrs={"data-item-id": True})

原因分析 #

Django视图函数运行在子线程中，而requests_html的render方法依赖异步事件循环，子线程默认没有初始化事件循环，导致报错。同时，普通requests无法处理JS动态渲染的内容，所以只能拿到初始加载的10个商品。

方案1：使用Playwright（推荐，更稳定可靠） #

Playwright是专门处理浏览器自动化的工具，能完整渲染JS生成的内容，适配大多数动态页面。

1. 安装依赖：

pip install playwright
playwright install chromium

2. 修改视图代码：

import requests, random
from django.shortcuts import render
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright


def wlista(request):
    buscarprods = request.COOKIES['buscarprod']
    url = 'https://www.walmart.com/search?q=hp+printers'  
    url = url.replace(" ", "%20")
    proveedor = request.COOKIES['proveedor'] 

    HDRS = {
       'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
       'Accept-Language': 'es-ES;es;q=0.8',
       'DNT': '1',
       'Connection': 'keep-alive',
       'Upgrade-Insecure-Requests': '1',
    }

    # 使用Playwright渲染页面
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)  # headless=False可查看浏览器窗口
        page = browser.new_page(extra_http_headers=HDRS)
        page.goto(url, wait_until="networkidle")  # 等待网络空闲，确保页面渲染完成
        r = page.content()  # 获取渲染后的完整HTML
        browser.close()

    soup = BeautifulSoup(r, "html.parser")
    rows = soup.find_all(attrs={"data-item-id": True})
    # 后续处理逻辑...
    return render(request, 'your_template.html', {'rows': rows})

方案2：修复requests_html的事件循环问题 #

如果坚持使用requests_html，需要在子线程中手动初始化事件循环：

import requests, random
import asyncio
from django.shortcuts import render
from bs4 import BeautifulSoup
from requests_html import AsyncHTMLSession


def wlista(request):
    buscarprods = request.COOKIES['buscarprod']
    url = 'https://www.walmart.com/search?q=hp+printers'  
    url = url.replace(" ", "%20")
    proveedor = request.COOKIES['proveedor'] 

    HDRS = {
       'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
       'Accept-Language': 'es-ES;es;q=0.8',
       'DNT': '1',
       'Connection': 'keep-alive',
       'Upgrade-Insecure-Requests': '1',
    }

    # 手动创建事件循环并运行异步请求
    def get_rendered_html():
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        asession = AsyncHTMLSession(loop=loop)
        r = loop.run_until_complete(asession.get(url, headers=HDRS))
        loop.run_until_complete(r.html.render(sleep=2))
        html = r.html.html
        asession.close()
        loop.close()
        return html

    r = get_rendered_html()

    soup = BeautifulSoup(r, "html.parser")
    rows = soup.find_all(attrs={"data-item-id": True})
    # 后续处理逻辑...
    return render(request, 'your_template.html', {'rows': rows})

注意：requests_html依赖Pyppeteer，安装和维护成本较高，且稳定性不如Playwright，优先推荐方案1。

内容的提问来源于stack exchange，提问作者Javier L. Camacaro