如何修改Python爬虫脚本获取WHO网站纯英文多语言出版物数据

我编写了一段Python脚本抓取WHO官网（iris.who.int）的数据，需求是从已设置过滤条件的父页面中提取标题、作者姓名、日期、PDF链接及子页面链接。目前脚本已能正常导出至CSV文件，但大量条目自动变为阿拉伯语等非英文版本——即便网站存在对应英文内容，PDF链接也会跳转到非英文版本。我认为脚本发送请求时获取到了非英文内容，希望修改脚本以确保仅获取英文数据，但不想通过过滤方式处理（避免丢失数据），需让脚本自动找到页面上对应的英文版本。

现有脚本代码：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
from http.client import RemoteDisconnected

BASE_URL = "https://iris.who.int"
PAGE_COUNT = 51  # Total pages
RPP = 100  # Results per page

# List of user-agents
USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; rv:90.0) Gecko/20100101 Firefox/90.0",
]

session = requests.Session()

def get_with_retries(url, max_retries=3):
    for attempt in range(max_retries):
        try:
            headers = {
                "User-Agent": random.choice(USER_AGENTS)
            }
            res = session.get(url, headers=headers, timeout=20)
            if res.status_code == 429:
                wait = 30 + random.randint(10, 30)
                print(f"⚠️  Rate limit hit. Waiting {wait}s...")
                time.sleep(wait)
                continue
            res.raise_for_status()
            return res
        except RemoteDisconnected:
            wait = 10 + attempt * 10
            print(f"[!] RemoteDisconnected — retrying in {wait}s...")
            time.sleep(wait)
        except requests.exceptions.RequestException as e:
            print(f"[!] Attempt {attempt + 1}/{max_retries} failed for {url}: {e}")
            time.sleep(10 * (attempt + 1))  # exponential backoff
    return None

def extract_details(item):
    # Check if language attribute is English
    lang = item.get("lang", "en")
    if lang != "en":
        return None  # Skip non-English

    title_tag = item.find("h4", class_="artifact-title")
    title = title_tag.get_text(strip=True) if title_tag else "N/A"

    author_tag = item.find("span", class_="author")
    if author_tag and author_tag.get("lang") and author_tag["lang"] != "en":
        return None  # Skip if author explicitly marked non-English
    author = author_tag.get_text(strip=True) if author_tag else "N/A"

    date_tag = item.find("span", class_="govdoc-date")
    dates = date_tag.get_text(strip=True) if date_tag else "N/A"

    href = item.find("a", href=True)
    child_url = BASE_URL + href["href"] if href else "N/A"

    return title, author, dates, child_url

def extract_pdf_link(child_url):
    response = get_with_retries(child_url)
    if not response:
        return None

    soup = BeautifulSoup(response.content, 'html.parser')

    # Prefer links that are clearly English
    pdf_links = soup.find_all("a", href=lambda x: x and ".pdf" in x)

    for link in pdf_links:
        href = link['href']
        text = link.get_text(strip=True).lower()
        if "english" in text or "en" in href:
            return BASE_URL + href

    # fallback if no clear English link
    if pdf_links:
        return BASE_URL + pdf_links[0]['href']

    return None

# Main scraping loop
data = []

for page_num in range(PAGE_COUNT):
    print(f"\n🔍 Scraping page {page_num + 1}/{PAGE_COUNT}...")
    url = f"https://iris.who.int/discover?filtertype=dateIssued&filter_relational_operator=equals&filter=2025&rpp={RPP}&page={page_num}"
    res = get_with_retries(url)
    if not res:
        print(f"[!] Skipping page {page_num + 1} due to repeated failures.")
        continue

    soup = BeautifulSoup(res.text, "html.parser")
    items = soup.select(".ds-artifact-item")

    for item in items:
        result = extract_details(item)
        if not result:
            continue  # Skip non-English

        title, author, dates, child_url = result
        pdf_link = extract_pdf_link(child_url) if child_url != "N/A" else "N/A"

        if pdf_link is None:
            continue  # Skip if no valid PDF found

        row = {
            "Title": title,
            "Author": author,
            "Dates": dates,
            "PDF_Link": pdf_link,
            "Child_Page": child_url
        }
        data.append(row)
        print(f"✅ Added: {title[:60]}...")

        time.sleep(random.uniform(2, 5))  # Random polite pause between items

    time.sleep(random.uniform(4, 8))  # Random pause between pages

# Save to file
df = pd.DataFrame(data)
df.to_csv("who_data_english_only.csv", index=False, encoding="utf-8-sig")
print(f"\n✅ Done! {len(df)} entries saved to who_data_english_only.csv")

核心修改方向 #

1. 请求头指定语言偏好：告诉服务器优先返回英文内容，从根源减少非英文条目
2. 子页面自动切换到英文版本：如果仍出现非英文页面，自动定位语言切换入口跳转后再提取数据
3. 移除原过滤逻辑：不再直接跳过非英文条目，而是主动获取对应英文版本

修改后的完整脚本 #

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
from http.client import RemoteDisconnected

BASE_URL = "https://iris.who.int"
PAGE_COUNT = 51  # Total pages
RPP = 100  # Results per page

# List of user-agents
USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; rv:90.0) Gecko/20100101 Firefox/90.0",
]

session = requests.Session()

def get_with_retries(url, max_retries=3, force_english=True):
    for attempt in range(max_retries):
        try:
            headers = {
                "User-Agent": random.choice(USER_AGENTS)
            }
            # 添加语言偏好头，强制优先返回英文
            if force_english:
                headers["Accept-Language"] = "en-US,en;q=0.9"
            res = session.get(url, headers=headers, timeout=20)
            if res.status_code == 429:
                wait = 30 + random.randint(10, 30)
                print(f"⚠️  Rate limit hit. Waiting {wait}s...")
                time.sleep(wait)
                continue
            res.raise_for_status()
            return res
        except RemoteDisconnected:
            wait = 10 + attempt * 10
            print(f"[!] RemoteDisconnected — retrying in {wait}s...")
            time.sleep(wait)
        except requests.exceptions.RequestException as e:
            print(f"[!] Attempt {attempt + 1}/{max_retries} failed for {url}: {e}")
            time.sleep(10 * (attempt + 1))  # exponential backoff
    return None

def get_english_child_url(child_url):
    # 获取子页面并检查是否有英文版本入口
    response = get_with_retries(child_url)
    if not response:
        return child_url
    
    soup = BeautifulSoup(response.content, 'html.parser')
    # 查找语言切换链接：通常带有lang=en参数或文本为English
    lang_switch_links = soup.find_all("a", href=lambda x: x and ("lang=en" in x or "English" in x.text.strip()))
    
    if lang_switch_links:
        # 取第一个有效英文链接
        english_href = lang_switch_links[0]['href']
        if not english_href.startswith("http"):
            english_href = BASE_URL + english_href
        print(f"🔄 Switched to English version: {english_href}")
        return english_href
    # 如果没有切换入口，返回原链接
    return child_url

def extract_details(item):
    title_tag = item.find("h4", class_="artifact-title")
    title = title_tag.get_text(strip=True) if title_tag else "N/A"

    author_tag = item.find("span", class_="author")
    author = author_tag.get_text(strip=True) if author_tag else "N/A"

    date_tag = item.find("span", class_="govdoc-date")
    dates = date_tag.get_text(strip=True) if date_tag else "N/A"

    href = item.find("a", href=True)
    child_url = BASE_URL + href["href"] if href else "N/A"

    # 自动获取对应英文子页面链接
    if child_url != "N/A":
        child_url = get_english_child_url(child_url)

    return title, author, dates, child_url

def extract_pdf_link(child_url):
    response = get_with_retries(child_url)
    if not response:
        return None

    soup = BeautifulSoup(response.content, 'html.parser')

    # 优先找明确标注英文的PDF链接
    pdf_links = soup.find_all("a", href=lambda x: x and ".pdf" in x)

    for link in pdf_links:
        href = link['href']
        text = link.get_text(strip=True).lower()
        if "english" in text or "en" in href:
            full_link = href if href.startswith("http") else BASE_URL + href
            return full_link

    # fallback到第一个PDF链接（此时页面已为英文，链接大概率是英文版本）
    if pdf_links:
        full_link = pdf_links[0]['href'] if pdf_links[0]['href'].startswith("http") else BASE_URL + pdf_links[0]['href']
        return full_link

    return None

# Main scraping loop
data = []

for page_num in range(PAGE_COUNT):
    print(f"\n🔍 Scraping page {page_num + 1}/{PAGE_COUNT}...")
    url = f"https://iris.who.int/discover?filtertype=dateIssued&filter_relational_operator=equals&filter=2025&rpp={RPP}&page={page_num}"
    res = get_with_retries(url)
    if not res:
        print(f"[!] Skipping page {page_num + 1} due to repeated failures.")
        continue

    soup = BeautifulSoup(res.text, "html.parser")
    items = soup.select(".ds-artifact-item")

    for item in items:
        title, author, dates, child_url = extract_details(item)
        
        if child_url == "N/A":
            pdf_link = "N/A"
        else:
            pdf_link = extract_pdf_link(child_url)

        row = {
            "Title": title,
            "Author": author,
            "Dates": dates,
            "PDF_Link": pdf_link if pdf_link else "N/A",
            "Child_Page": child_url
        }
        data.append(row)
        print(f"✅ Added: {title[:60]}...")

        time.sleep(random.uniform(2, 5))  # Random polite pause between items

    time.sleep(random.uniform(4, 8))  # Random pause between pages

# Save to file
df = pd.DataFrame(data)
df.to_csv("who_data_english_only.csv", index=False, encoding="utf-8-sig")
print(f"\n✅ Done! {len(df)} entries saved to who_data_english_only.csv")

关键修改说明 #

• 请求头新增Accept-Language：在get_with_retries中添加该头，让服务器优先返回英文内容，从源头减少非英文条目
• 新增get_english_child_url函数：自动检测子页面的语言切换入口，跳转到英文版本页面，确保后续提取的是英文内容
• 移除原过滤逻辑：不再跳过非英文条目，而是主动获取对应英文版本，避免丢失数据
• 优化PDF链接提取：由于子页面已切换为英文， fallback的PDF链接也会是英文版本，无需额外过滤

内容的提问来源于stack exchange，提问作者Mann Jain