如何用Python从谷歌搜索提取Sherdog相关URL并写入DataFrame

需求说明 #

读取指定CSV文件，对每行的name_a、name_b字段值分别拼接"sherdog"作为关键词执行谷歌搜索，从搜索结果中提取首个包含sherdog.com的URL，将结果写入DataFrame对应的url_a、url_b列。

实现代码 #

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager

# 读取CSV文件，创建包含'name_a', 'url_a', 'name_b', 'url_b'列的DataFrame
df = pd.read_csv('/Users/daneweickert/Downloads/Sherdog Links - UFC 285_ Jones vs Gane (1).csv')
print(df)

# 配置Chrome浏览器驱动
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # 无头模式运行，不显示浏览器窗口
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)

# 遍历DataFrame的每一行
for index, row in df.iterrows():
    name_a = row['name_a']
    
    # 以"name_a + sherdog"为关键词执行谷歌搜索
    query = f"{name_a} sherdog"
    url_a = ''
    try:
        driver.get(f"https://www.google.com/search?q={query}")
        print(f"搜索关键词: {query}")
        # 获取首个来自sherdog.com的搜索结果链接
        top_result = driver.find_element(By.CSS_SELECTOR,'div.g div.r a[href^="https://www.sherdog.com/"]')
        # 也可以用XPATH定位：top_result = driver.find_element(By.XPATH,'//a[@href^="https://www.sherdog.com/"]')
        url_a = top_result.get_attribute('href')
        print(f"{name_a}的匹配链接: {url_a}")
    except Exception as e:
        print(f"搜索{name_a}时出错: {str(e)}")
    
    # 更新当前行的url_a列
    df.at[index, 'url_a'] = url_a
    
    # 注意：原代码此处的内层循环会重复处理后续行的name_b，建议修改为仅处理当前行的name_b
    # 修正后的处理方式：
    # name_b = row['name_b']
    # query = f"{name_b} sherdog"
    # url_b = ''
    # try:
    #     driver.get(f"https://www.google.com/search?q={query}")
    #     print(f"搜索关键词: {query}")
    #     top_result = driver.find_element(By.CSS_SELECTOR,'div.g div.r a[href^="https://www.sherdog.com/"]')
    #     url_b = top_result.get_attribute('href')
    #     print(f"{name_b}的匹配链接: {url_b}")
    # except Exception as e:
    #     print(f"搜索{name_b}时出错: {str(e)}")
    # df.at[index, 'url_b'] = url_b

    # 原代码的name_b处理逻辑（存在重复执行问题）
    for i in range(index, len(df)):
        name_b = df.at[i, 'name_b']
        query = f"{name_b} sherdog"
        url_b = ''
        try:
            driver.get(f"https://www.google.com/search?q={query}")
            print(f"搜索关键词: {query}")
            # 获取首个来自sherdog.com的搜索结果链接
            top_result = driver.find_element(By.CSS_SELECTOR,'div.g div.r a[href^="https://www.sherdog.com/"]')
            # 也可以用XPATH定位：top_result = driver.find_element(By.XPATH,'//a[@href^="https://www.sherdog.com/"]')
            url_b = top_result.get_attribute('href')
            print(f"{name_b}的匹配链接: {url_b}")
        except Exception as e:
            print(f"搜索{name_b}时出错: {str(e)}")
        
        # 更新当前行的url_b列
        df.at[i, 'url_b'] = url_b

# 关闭浏览器驱动
driver.quit()

print(df)

注意事项 #

• 运行代码前需要安装依赖库：pip install pandas selenium webdriver-manager
• 代码采用无头Chrome模式运行，不会弹出浏览器窗口，若需要可视化调试可注释掉--headless参数
• 原代码中处理name_b的内层循环存在逻辑缺陷，会对后续行的name_b重复执行搜索操作，建议替换为注释中的修正逻辑，仅处理当前行的name_b

内容的提问来源于stack exchange，提问作者Dalej400