如何从嵌套div中爬取ESA图库的高清图片及元数据？

问题分析 #

你的代码仅能抓取单页缩略图跳转链接，存在三个核心问题：

• 未处理图库的分页结构，无法获取多页内容
• 仅提取了页面展示的压缩图，而非最高清版本
• 未实现图片描述、来源元数据的抓取

完整解决方案代码 #

import requests
from bs4 import BeautifulSoup
import os
import time

base_url = "https://www.esa.int"
main_url = "https://www.esa.int/ESA_Multimedia/Sets/Earth_from_Space_image_collection/(result_type)/images"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
}
save_dir = "ESA_Earth_Images"
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

def extract_image_data(page_url):
    """提取单页内所有图片的跳转链接"""
    response = requests.get(page_url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    return [base_url + a['href'] for a in soup.find_all('a', class_='fancybox')]

def get_full_image_info(img_page_url):
    """提取高清图链接、描述、元数据"""
    response = requests.get(img_page_url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    
    # 提取高清图片链接（优先取下载区域的最高清版本）
    full_img_url = None
    download_links = soup.find_all('a', class_='download')
    for link in download_links:
        if 'Full resolution' in link.text:
            full_img_url = base_url + link['href']
            break
    # 若找不到高清下载链接，退而求其次取页面展示图
    if not full_img_url:
        img_tag = soup.find('div', class_='image').find('img')
        if img_tag:
            full_img_url = base_url + img_tag['src']
    
    # 提取图片描述
    description = ""
    desc_div = soup.find('div', class_='text')
    if desc_div:
        description = desc_div.get_text(strip=True, separator='\n')
    
    # 提取来源元数据（如拍摄时间、卫星信息）
    metadata = {}
    meta_items = soup.find_all('div', class_='metadata-item')
    for item in meta_items:
        label = item.find('span', class_='label').get_text(strip=True)
        value = item.find('span', class_='value').get_text(strip=True)
        metadata[label] = value
    
    return full_img_url, description, metadata

def download_image(img_url, save_path):
    """下载图片到指定路径"""
    try:
        img_data = requests.get(img_url, headers=headers).content
        with open(save_path, 'wb') as f:
            f.write(img_data)
        return True
    except Exception as e:
        print(f"下载失败 {img_url}: {str(e)}")
        return False

# 处理分页抓取
current_page_url = main_url
page_num = 1
while current_page_url:
    print(f"正在处理第 {page_num} 页...")
    thumb_links = extract_image_data(current_page_url)
    
    for idx, link in enumerate(thumb_links, 1):
        print(f"正在处理图片 {idx}/{len(thumb_links)}")
        full_img_url, description, metadata = get_full_image_info(link)
        
        if not full_img_url:
            print(f"无法获取图片链接: {link}")
            continue
        
        # 生成文件名（用元数据里的标题或编号，避免重复）
        img_name = f"page{page_num}_img{idx}"
        if 'Title' in metadata:
            img_name = metadata['Title'].replace('/', '_').replace('\\', '_')
        img_path = os.path.join(save_dir, f"{img_name}.jpg")
        
        # 下载图片
        if download_image(full_img_url, img_path):
            # 保存描述和元数据到文本文件
            meta_path = os.path.join(save_dir, f"{img_name}_info.txt")
            with open(meta_path, 'w', encoding='utf-8') as f:
                f.write("图片描述:\n")
                f.write(description + "\n\n")
                f.write("元数据:\n")
                for k, v in metadata.items():
                    f.write(f"{k}: {v}\n")
        
        # 加延迟，避免请求过于频繁被封
        time.sleep(1)
    
    # 获取下一页链接
    response = requests.get(current_page_url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    next_page = soup.find('a', class_='next')
    if next_page and 'href' in next_page.attrs:
        current_page_url = base_url + next_page['href']
        page_num += 1
        time.sleep(2)
    else:
        current_page_url = None

print("所有图片及信息抓取完成！")

关键优化说明 #

• 分页处理：通过查找页面的next类链接，循环抓取所有分页内容
• 高清图获取：优先提取页面下载区域的「Full resolution」高清链接，而非页面展示的压缩图
• 元数据与描述提取：解析页面的text类区块获取描述，metadata-item类区块提取拍摄时间、卫星、标题等元数据
• 反爬优化：添加User-Agent模拟浏览器请求，设置请求延迟避免被服务器拦截
• 文件命名优化：用图片标题（替换非法字符）命名文件，同时保存对应的描述和元数据文本

内容的提问来源于stack exchange，提问作者Christopher Phillips