如何使用Python抓取多网页并将英文内容翻译为印地语？

现有代码可正常运行无报错，但仅能抓取目标网站主页面的特定文本并翻译成印地语，需要实现抓取该网站多个页面并将每个页面内容全部翻译为印地语的功能。

原抓取脚本和翻译脚本如下：

原网页抓取脚本 #

#Script scraps the website using request and beautifulSoup library

from google_translate import browser
from google_translate import selenium
import requests
from bs4 import BeautifulSoup
URL = "https://www.classcentral.com/?"
headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"}
# Here the user agent is for Edge browser on windows 10. You can find your browser user agent from the above given link.
r = requests.get(url=URL, headers=headers)
print(r.content)
# Parsing the HTML
soup = BeautifulSoup(r.content, 'html.parser')
# find all the anchor tags with "href"
for link in soup.find_all('a'):
    print(link.get('href'))

原翻译脚本 #

#Script transalate text into Hindi using google translate API

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
import selenium
# Give Language code in which you want to translate the text:=>
lang_code = 'hi '

# Provide text that you want to translate:=>
input1 = " Find your next course.Class Central aggregates courses from many providers to help you find the best courses on almost any subject, wherever they exist"

# launch browser with selenium:=>
browser = webdriver.Chrome() #browser = webdriver.Chrome('path of chromedriver.exe file') if the chromedriver.exe is in different folder

# copy google Translator link here:=>
browser.get("https://translate.google.co.in/?sl=auto&tl="+lang_code+"&text="+input1+"&op=translate")

# just wait for some time for translating input text:=>
time.sleep(6)

# Given below x path contains the translated output that we are storing in output variable:=>
output1 = browser.find_element(By.CLASS_NAME,'HwtZe').text

# Display the output:=>
print("Translated Paragraph:=> " + output1)

核心改进点 #

• 过滤有效内部链接，去重并处理相对路径
• 提取页面正文内容（而非仅链接）
• 整合抓取与翻译流程，批量处理多页面
• 替换固定延迟为显式等待，提升翻译稳定性

整合后的完整代码 #

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from urllib.parse import urljoin
import time

# 配置项
BASE_URL = "https://www.classcentral.com/"
HEADERS = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"}
TARGET_LANG = 'hi'
MAX_PAGES = 5  # 限制抓取页面数量，避免爬取过多

def get_internal_links(url):
    """获取页面中所有有效的内部链接并去重"""
    try:
        response = requests.get(url, headers=HEADERS)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        links = set()
        for a_tag in soup.find_all('a', href=True):
            href = a_tag['href']
            # 过滤外部链接和无效链接
            if href.startswith('/') and not href.startswith('//'):
                full_url = urljoin(BASE_URL, href)
                links.add(full_url)
            elif href.startswith(BASE_URL):
                links.add(href)
        return list(links)[:MAX_PAGES]  # 返回指定数量的链接
    except Exception as e:
        print(f"获取链接失败: {e}")
        return []

def extract_page_content(url):
    """提取页面的正文文本内容"""
    try:
        response = requests.get(url, headers=HEADERS)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        # 移除脚本、样式等无关标签
        for tag in soup(['script', 'style', 'nav', 'footer', 'aside']):
            tag.decompose()
        # 获取所有段落文本并拼接
        paragraphs = soup.find_all('p')
        content = '\n'.join([p.get_text(strip=True) for p in paragraphs if p.get_text(strip=True)])
        return content if content else "页面无有效文本内容"
    except Exception as e:
        print(f"提取页面内容失败 {url}: {e}")
        return ""

def translate_text(browser, text, target_lang):
    """使用谷歌翻译将文本翻译成目标语言"""
    try:
        # 构造翻译URL，处理文本中的特殊字符
        encoded_text = requests.utils.quote(text)
        translate_url = f"https://translate.google.co.in/?sl=auto&tl={target_lang}&text={encoded_text}&op=translate"
        browser.get(translate_url)
        # 显式等待翻译结果加载完成
        wait = WebDriverWait(browser, 10)
        result_element = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'HwtZe')))
        return result_element.text
    except Exception as e:
        print(f"翻译失败: {e}")
        return ""

def main():
    # 初始化浏览器
    browser = webdriver.Chrome()
    try:
        # 获取主页面的内部链接
        internal_links = get_internal_links(BASE_URL)
        print(f"共获取到 {len(internal_links)} 个页面待处理")
        
        for idx, page_url in enumerate(internal_links, 1):
            print(f"\n=== 处理第 {idx} 个页面: {page_url} ===")
            # 提取页面内容
            content = extract_page_content(page_url)
            print("原始内容预览:", content[:200] + "..." if len(content) > 200 else content)
            
            # 翻译内容
            translated_content = translate_text(browser, content, TARGET_LANG)
            print("翻译内容预览:", translated_content[:200] + "..." if len(translated_content) > 200 else translated_content)
            
            # 可选：将翻译结果保存到文件
            with open(f"translated_page_{idx}.txt", 'w', encoding='utf-8') as f:
                f.write(f"原始页面URL: {page_url}\n\n原始内容:\n{content}\n\n翻译内容:\n{translated_content}")
            print(f"结果已保存到 translated_page_{idx}.txt")
            
            # 页面间添加短暂延迟，避免被限制
            time.sleep(2)
    finally:
        # 关闭浏览器
        browser.quit()

if __name__ == "__main__":
    main()

代码说明 #

1. 链接处理：get_internal_links函数过滤出网站内部链接，使用集合去重，避免重复爬取相同页面
2. 内容提取：extract_page_content函数移除页面无关标签，仅保留正文段落文本
3. 翻译优化：使用WebDriverWait显式等待翻译结果加载，比固定延迟更稳定；对文本进行URL编码，避免特殊字符导致的URL错误
4. 批量处理：循环处理每个页面，支持将结果保存到本地文件，方便查看

内容的提问来源于stack exchange，提问作者Themba2023 Bhele