如何用Python+DataFrame统计网页URL数量并实现文本英译印地语？

1. 完善爬取逻辑并创建DataFrame #

你的现有代码仅抓取了课程标题，要统计URL数量需同时提取课程对应的链接。以下是修改后的爬取代码，直接生成包含标题和URL的DataFrame：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

# 初始化浏览器并访问目标页面
url = "https://www.classcentral.com/collection/top-free-online-courses"
driver = webdriver.Chrome()
driver.get(url)

# 用显式等待替代time.sleep，提升稳定性
wait = WebDriverWait(driver, 10)
all_courses = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'catalog-grid__results')))

# 批量提取课程标题和对应URL
course_elements = all_courses.find_elements(By.CSS_SELECTOR, '[class="color-charcoal course-name"]')
course_data = []
for elem in course_elements:
    course_data.append({
        "title": elem.text,
        "url": elem.get_attribute('href')
    })

# 创建DataFrame
df = pd.DataFrame(course_data)
print(df.head())

# 关闭浏览器释放资源
driver.quit()

2. 编写URL统计函数 #

基于DataFrame实现URL数量统计，包含总条目数、有效URL数和缺失URL数：

def count_course_urls(df):
    total_entries = len(df)
    valid_url_count = df['url'].notna().sum()
    return {
        "总课程条目数": total_entries,
        "有效URL数量": valid_url_count,
        "缺失URL数量": total_entries - valid_url_count
    }

# 调用函数查看结果
url_stats = count_course_urls(df)
print("URL统计结果：")
for key, value in url_stats.items():
    print(f"{key}: {value}")

3. 英文标题转印地语 #

使用googletrans库实现翻译（先执行pip install googletrans==4.0.0-rc1安装指定版本，避免API兼容问题）：

from googletrans import Translator

def translate_to_hindi(df, text_column='title'):
    translator = Translator()
    # 批量翻译课程标题
    df['hindi_title'] = df[text_column].apply(lambda x: translator.translate(x, dest='hi').text)
    return df

# 执行翻译并预览结果
df_with_translation = translate_to_hindi(df)
print(df_with_translation[['title', 'hindi_title']].head())

完整整合代码 #

将所有功能整合为可直接运行的脚本：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from googletrans import Translator

def scrape_class_central_courses(url):
    driver = webdriver.Chrome()
    driver.get(url)
    wait = WebDriverWait(driver, 10)
    
    all_courses = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'catalog-grid__results')))
    course_elements = all_courses.find_elements(By.CSS_SELECTOR, '[class="color-charcoal course-name"]')
    
    course_data = []
    for elem in course_elements:
        course_data.append({
            "title": elem.text,
            "url": elem.get_attribute('href')
        })
    
    driver.quit()
    return pd.DataFrame(course_data)

def count_course_urls(df):
    total = len(df)
    valid = df['url'].notna().sum()
    return {
        "总课程数": total,
        "有效URL数": valid,
        "缺失URL数": total - valid
    }

def translate_to_hindi(df):
    translator = Translator()
    df['hindi_title'] = df['title'].apply(lambda x: translator.translate(x, dest='hi').text)
    return df

# 主执行流程
if __name__ == "__main__":
    target_url = "https://www.classcentral.com/collection/top-free-online-courses"
    course_df = scrape_class_central_courses(target_url)
    
    # 统计URL
    url_stats = count_course_urls(course_df)
    print("URL统计：")
    for k, v in url_stats.items():
        print(f"{k}: {v}")
    
    # 翻译标题
    course_df = translate_to_hindi(course_df)
    print("\n翻译后数据预览：")
    print(course_df[['title', 'hindi_title', 'url']].head())

注意事项 #

• 若googletrans出现连接异常，可尝试更换网络环境，或改用translate等其他翻译库
• Selenium驱动需与本地Chrome版本匹配，避免启动失败
• 大规模爬取时建议增加请求间隔，降低被网站反爬限制的风险

内容的提问来源于stack exchange，提问作者user21319072