如何用Python实现类似Firefox的完整网页保存功能？

现有代码的缺陷 #

你提供的代码仅处理了img、link（仅href）、script（仅src）标签的资源，但Firefox的「另存为」会覆盖更多场景：

• CSS文件中引用的背景图、字体、图标资源
• 媒体标签（如video、audio、source的src/srcset）
• 内联样式中引用的资源
• 文件名重复时的覆盖问题

最佳实现方案 #

方案1：改进现有代码，覆盖全场景资源 #

针对现有代码的不足，扩展处理逻辑，支持HTML多标签资源、CSS内部资源引用，并优化文件名冲突问题：

import os
import re
import sys
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
from cssutils import parseString

def saveFullHtmlPage(url, pagepath='page', session=requests.Session(), html=None):
    """保存网页及关联静态资源，模拟Firefox「另存为」效果
        * pagepath : 保存路径前缀
        会生成 `pagepath.html` 文件和 `pagepath_files` 子文件夹
    """
    def get_unique_filename(folder, filename):
        # 处理文件名重复，避免覆盖
        base, ext = os.path.splitext(filename)
        counter = 1
        new_filename = filename
        while os.path.exists(os.path.join(folder, new_filename)):
            new_filename = f"{base}_{counter}{ext}"
            counter += 1
        return new_filename

    def save_resource(fileurl, filepath, session):
        # 统一资源下载逻辑，添加异常处理
        try:
            response = session.get(fileurl, stream=True)
            response.raise_for_status()
            with open(filepath, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    f.write(chunk)
            return True
        except Exception as exc:
            print(f"下载资源失败 {fileurl}: {exc}", file=sys.stderr)
            return False

    def process_html_resources(soup, pagefolder, session, url):
        # 处理HTML各类标签的资源属性
        tag_attr_map = {
            'img': ['src', 'srcset'],
            'link': ['href'],
            'script': ['src'],
            'video': ['src'],
            'audio': ['src'],
            'source': ['src', 'srcset'],
            'iframe': ['src']  # 不需要保存iframe内容可注释此行
        }

        for tag, attrs in tag_attr_map.items():
            for element in soup.find_all(tag):
                for attr in attrs:
                    if not element.has_attr(attr):
                        continue
                    resource_url = element[attr]
                    abs_url = urljoin(url, resource_url)
                    # 跳过非HTTP/HTTPS链接
                    if not abs_url.startswith(('http://', 'https://')):
                        continue
                    
                    # 清理文件名，去掉URL参数和非法字符
                    filename = os.path.basename(abs_url.split('?')[0])
                    filename = re.sub(r'[^\w\.-]', '', filename)
                    filename = get_unique_filename(pagefolder, filename)
                    filepath = os.path.join(pagefolder, filename)
                    
                    # 更新HTML中的资源引用路径
                    element[attr] = os.path.join(os.path.basename(pagefolder), filename)
                    
                    if not os.path.exists(filepath):
                        save_resource(abs_url, filepath, session)

    def process_css_resources(css_content, pagefolder, session, base_url):
        # 处理CSS内部引用的资源（背景图、字体等）
        updated_css = css_content
        # 匹配CSS中的url()语法
        url_pattern = re.compile(r'url\(["\']?(.*?)["\']?\)', re.IGNORECASE)
        
        for match in url_pattern.finditer(css_content):
            resource_url = match.group(1)
            abs_url = urljoin(base_url, resource_url)
            if not abs_url.startswith(('http://', 'https://')):
                continue
            
            filename = os.path.basename(abs_url.split('?')[0])
            filename = re.sub(r'[^\w\.-]', '', filename)
            filename = get_unique_filename(pagefolder, filename)
            filepath = os.path.join(pagefolder, filename)
            
            # 更新CSS中的资源引用路径
            new_path = os.path.join(os.path.basename(pagefolder), filename)
            updated_css = updated_css.replace(match.group(1), new_path)
            
            if not os.path.exists(filepath):
                save_resource(abs_url, filepath, session)
        
        return updated_css

    # 创建资源存储文件夹
    path, _ = os.path.splitext(pagepath)
    pagefolder = f"{path}_files"
    if not os.path.exists(pagefolder):
        os.makedirs(pagefolder)

    # 获取目标页面HTML
    if not html:
        response = session.get(url)
        response.raise_for_status()
        html = response.text
    soup = BeautifulSoup(html, "html.parser")

    # 处理HTML标签中的资源
    process_html_resources(soup, pagefolder, session, url)

    # 处理内联CSS样式
    for style_tag in soup.find_all('style'):
        if style_tag.string:
            updated_css = process_css_resources(style_tag.string, pagefolder, session, url)
            style_tag.string = updated_css

    # 处理外部CSS文件（下载→解析修改→保存→更新HTML引用）
    for link_tag in soup.find_all('link', rel='stylesheet'):
        if not link_tag.has_attr('href'):
            continue
        css_url = urljoin(url, link_tag['href'])
        css_filename = os.path.basename(css_url.split('?')[0])
        css_filename = re.sub(r'[^\w\.-]', '', css_filename)
        css_filename = get_unique_filename(pagefolder, css_filename)
        css_filepath = os.path.join(pagefolder, css_filename)
        
        if not os.path.exists(css_filepath):
            response = session.get(css_url)
            if response.status_code == 200:
                css_content = response.text
                updated_css = process_css_resources(css_content, pagefolder, session, css_url)
                with open(css_filepath, 'w', encoding='utf-8') as f:
                    f.write(updated_css)
                # 更新HTML中的CSS引用路径
                link_tag['href'] = os.path.join(os.path.basename(pagefolder), css_filename)

    # 保存修改后的HTML文件
    with open(f"{path}.html", 'wb') as f:
        f.write(soup.prettify('utf-8'))

使用前需安装依赖：

pip install beautifulsoup4 requests cssutils

方案2：使用成熟第三方库pywebcopy #

如果不想手动实现复杂逻辑，可使用专门的网页保存库pywebcopy，它能模拟浏览器保存行为，自动处理资源抓取和路径重写：

from pywebcopy import save_webpage

def save_page_with_pywebcopy(url, save_folder):
    # 配置仅保存主页面及关联静态资源，不跟进外部链接
    kwargs = {
        'url': url,
        'project_folder': save_folder,
        'project_name': 'saved_page',
        'bypass_robots': True,
        'recursive': False,  # 禁止递归爬取外部链接
        'open_in_browser': False,
        'threaded': False,  # 批量处理时可开启线程，注意控制并发量
    }
    save_webpage(**kwargs)

安装依赖：

pip install pywebcopy

为什么不选Selenium？ #

Selenium需要启动浏览器实例，资源占用高、处理速度慢，完全不适合批量处理大量页面的场景，而基于HTTP请求+静态解析的方案更高效、轻量化。

内容的提问来源于stack exchange，提问作者Zee Kay