如何从iframe中提取MP4视频URL？（Python/Rust实现）

问题描述 #

需要用Python或Rust（不限库）从IFrame中提取MP4视频URL，示例IFrame如下：

<iframe src="https://spinning.allohalive.com/?kp=1332827&token=b51bdfc8af17dee996d3eae53726df" />

已尝试Bs4、Selenium、Scrapy等工具，耗时两周仍未成功，现有用于解析目标网站IFrame的Scrapy代码如下：

import scrapy

from cimber.models.website import Website


class KinokradSpider(scrapy.Spider):
    name = "kinokrad"
    start_urls = [Website.Kinokrad.value]

    def __init__(self):
        self.pages_count = 1

    def parse(self, response):
        pages_count = self.get_pages_count(response)

        if self.pages_count <= pages_count:
            for film in response.css("div.shorposterbox"):
                film_url = film.css("div.postertitle").css("a").attrib["href"]
                yield scrapy.Request(film_url, callback=self.parse_film)

            next_page = f"{Website.Kinokrad.value}/page/{self.pages_count}"
            if next_page is not None:
                yield response.follow(next_page, callback=self.parse)

            self.pages_count += 1

    def parse_film(self, response):
        name = response.css("div.fallsttitle").css("h1::text").get().strip()
        players = []

        for player in  response.css("iframe::attr(src)").extract():
            players.append(player)

        yield {
            "name": name,
            "players": players
        }

    def get_pages_count(self, response) -> int:
        links = response.css("div.navcent").css("a")
        last_link = links[len(links) - 1].attrib["href"]

        return int(last_link.split("/page/")[1].replace("/", "").strip())

解决方案思路 #

1. 核心问题分析 #

IFrame指向的页面大多通过JavaScript动态加载视频资源，静态爬取HTML无法直接拿到MP4 URL，必须处理动态渲染的内容：要么解析页面中的JS代码提取视频地址，要么模拟浏览器执行JS，等待视频加载后获取真实URL。

2. Python实现方案（Scrapy + Playwright） #

Playwright能模拟真实浏览器环境，执行页面JS并捕获动态加载的资源，适合解决这类问题。

步骤1：安装依赖

pip install scrapy playwright
playwright install chromium

步骤2：修改爬虫代码

在parse_film方法中，对每个IFrame地址发起浏览器请求，提取视频URL：

import scrapy
from playwright.sync_api import sync_playwright
from cimber.models.website import Website


class KinokradSpider(scrapy.Spider):
    name = "kinokrad"
    start_urls = [Website.Kinokrad.value]

    def __init__(self):
        self.pages_count = 1

    def parse(self, response):
        pages_count = self.get_pages_count(response)

        if self.pages_count <= pages_count:
            for film in response.css("div.shorposterbox"):
                film_url = film.css("div.postertitle").css("a").attrib["href"]
                yield scrapy.Request(film_url, callback=self.parse_film)

            next_page = f"{Website.Kinokrad.value}/page/{self.pages_count}"
            if next_page is not None:
                yield response.follow(next_page, callback=self.parse)

            self.pages_count += 1

    def parse_film(self, response):
        name = response.css("div.fallsttitle").css("h1::text").get().strip()
        players = response.css("iframe::attr(src)").extract()
        video_urls = []

        with sync_playwright() as p:
            browser = p.chromium.launch(headless=True)
            for player_url in players:
                page = browser.new_page(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
                # 等待页面加载完成，网络请求稳定
                page.goto(player_url, wait_until="networkidle")
                # 优先查找video标签的src属性，可根据实际页面调整选择器
                video_element = page.query_selector("video")
                if video_element:
                    video_src = video_element.get_attribute("src")
                    if video_src and video_src.endswith(".mp4"):
                        video_urls.append(video_src)
                # 若video标签无src，尝试解析页面中的JS变量（示例逻辑，需根据实际页面调整）
                else:
                    video_data = page.evaluate("""() => {
                        // 假设页面中有window.videoInfo这样的变量存储视频地址
                        return window.videoInfo?.src || null
                    }""")
                    if video_data:
                        video_urls.append(video_data)
                page.close()
            browser.close()

        yield {
            "name": name,
            "players": players,
            "video_urls": video_urls
        }

    def get_pages_count(self, response) -> int:
        links = response.css("div.navcent").css("a")
        last_link = links[len(links) - 1].attrib["href"]

        return int(last_link.split("/page/")[1].replace("/", "").strip())

3. 关键注意事项 #

• 目标网站可能有反爬机制，需添加真实的User-Agent，设置请求间隔，避免被封禁
• 不同视频页面的DOM结构或JS逻辑不同，需通过浏览器开发者工具分析：查看视频元素的属性、查找存储视频URL的JS变量，或追踪网络请求中MP4文件的地址
• 若遇到加密的视频URL，可能需要分析页面的解密逻辑，模拟生成有效地址

内容的提问来源于stack exchange，提问作者Александр Прокопенко