如何高效提取并处理经重定向链接获取的PDF文件？

你已经能抓取到一批指向含PDF页面的链接，下面是高效获取并处理这些PDF的实现方法：

现有代码与输出 #

现有代码 #

import requests
from bs4 import BeautifulSoup
import re

url = 'https://oeil.secure.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2014/0124(COD)&l=en'
reqs = requests.get(url)
soup = BeautifulSoup(reqs.text, 'html.parser')

urls = []
for link in soup.find_all("a",href=re.compile("AM")):
    print(link.get('href'))

现有输出 #

https://www.europarl.europa.eu/doceo/document/EMPL-AM-544465_EN.html
https://www.europarl.europa.eu/doceo/document/EMPL-AM-541655_EN.html
https://www.europarl.europa.eu/doceo/document/EMPL-AM-551891_EN.html
https://www.europarl.europa.eu/doceo/document/EMPL-AM-544465_EN.html
https://www.europarl.europa.eu/doceo/document/EMPL-AM-541655_EN.html
https://www.europarl.europa.eu/doceo/document/EMPL-AM-551891_EN.html

高效实现步骤 #

1. 先给链接去重 #

现有输出存在重复链接，先去重避免重复处理：

# 对收集到的链接去重
unique_urls = list(set(urls))

2. 抓取每个跳转页面中的PDF链接 #

遍历去重后的链接，进入对应页面提取PDF地址：

def get_pdf_links(page_url):
    req = requests.get(page_url)
    soup = BeautifulSoup(req.text, 'html.parser')
    # 筛选所有指向PDF的链接（匹配后缀为.pdf的地址）
    pdf_links = [a['href'] for a in soup.find_all('a', href=re.compile(r'\.pdf$', re.IGNORECASE))]
    # 补全相对链接的域名
    pdf_links = [link if link.startswith('http') else f"https://www.europarl.europa.eu{link}" for link in pdf_links]
    return pdf_links

# 收集所有PDF链接
all_pdf_links = []
for page_url in unique_urls:
    all_pdf_links.extend(get_pdf_links(page_url))

3. 下载并处理PDF #

用requests下载PDF，再用pdfplumber（文本提取更精准）提取内容。先安装依赖：

pip install requests beautifulsoup4 pdfplumber

处理示例代码：

import pdfplumber

def process_pdf(pdf_url):
    # 下载PDF文件
    response = requests.get(pdf_url)
    pdf_filename = pdf_url.split('/')[-1]
    # 保存到本地（也可直接在内存中处理，无需保存文件）
    with open(pdf_filename, 'wb') as f:
        f.write(response.content)
    
    # 提取PDF文本内容
    with pdfplumber.open(pdf_filename) as pdf:
        full_text = ''
        for page in pdf.pages:
            full_text += page.extract_text() or ''
        # 这里可添加自定义处理逻辑，比如保存文本、分析内容等
        print(f"处理完成：{pdf_filename}")
        return full_text

# 逐个处理所有PDF链接
for pdf_link in all_pdf_links:
    process_pdf(pdf_link)

4. 并行处理提升效率 #

网络请求和PDF处理属于IO密集型操作，用多线程可大幅提速：

from concurrent.futures import ThreadPoolExecutor

# 线程数根据网络情况调整
MAX_WORKERS = 5

with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
    executor.map(process_pdf, all_pdf_links)

完整整合代码 #

import requests
from bs4 import BeautifulSoup
import re
import pdfplumber
from concurrent.futures import ThreadPoolExecutor

def get_am_links(base_url):
    reqs = requests.get(base_url)
    soup = BeautifulSoup(reqs.text, 'html.parser')
    am_links = [link.get('href') for link in soup.find_all("a", href=re.compile("AM"))]
    # 去重返回
    return list(set(am_links))

def get_pdf_links(page_url):
    req = requests.get(page_url)
    soup = BeautifulSoup(req.text, 'html.parser')
    pdf_links = [a['href'] for a in soup.find_all('a', href=re.compile(r'\.pdf$', re.IGNORECASE))]
    # 补全域名
    return [link if link.startswith('http') else f"https://www.europarl.europa.eu{link}" for link in pdf_links]

def process_pdf(pdf_url):
    try:
        response = requests.get(pdf_url)
        response.raise_for_status()  # 检查请求是否成功
        pdf_filename = pdf_url.split('/')[-1]
        with open(pdf_filename, 'wb') as f:
            f.write(response.content)
        
        with pdfplumber.open(pdf_filename) as pdf:
            full_text = ''
            for page in pdf.pages:
                full_text += page.extract_text() or ''
        print(f"成功处理：{pdf_filename}")
        # 可选：将提取的文本保存为文件
        # with open(f"{pdf_filename}.txt", 'w', encoding='utf-8') as f:
        #     f.write(full_text)
    except Exception as e:
        print(f"处理失败 {pdf_url}: {str(e)}")

if __name__ == "__main__":
    base_url = 'https://oeil.secure.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2014/0124(COD)&l=en'
    am_links = get_am_links(base_url)
    
    all_pdf_links = []
    for page_url in am_links:
        all_pdf_links.extend(get_pdf_links(page_url))
    
    # 并行处理PDF
    with ThreadPoolExecutor(max_workers=5) as executor:
        executor.map(process_pdf, all_pdf_links)

内容的提问来源于stack exchange，提问作者Ann