使用Selenium爬取谷歌搜索结果邮箱并提取嵌套列表字符串

我通过Selenium WebDriver在谷歌搜索指定关键词，获取搜索结果链接后逐个访问，提取页面内的邮箱地址。目前得到的结果是嵌套列表格式，需要把其中的字符串类型邮箱提取出来，整理成单一的无重复字符串列表。

现有输出示例 #

[[], [], [], [], ['info@neurotechnology.com', 'support@neurotechnology.com', 'support@neurotechnology.com'], ['info@neurotechnology.com', 'support@neurotechnology.com', 'support@neurotechnology.com'], ['info@himalayakarya.com', 'info@himalayakarya.com'], ['info@himalayakarya.com', 'info@himalayakarya.com'], ['sales@obejor.com', 'sales@obejor.com'], ['sales@obejor.com', 'sales@obejor.com'], [], [], ['507c10c65fb14333bfa540a7cfb4b543@o417096.ingest.sentry.io'], ['507c10c65fb14333bfa540a7cfb4b543@o417096.ingest.sentry.io'], ['hello@paykobo.com'], ['hello@paykobo.com'], ['customercare@indiamart.com'], ['customercare@indiamart.com'], [], [], [], []]
[[], ['info@neurotechnology.com', 'support@neurotechnology.com', 'support@neurotechnology.com'], ['info@himalayakarya.com', 'info@himalayakarya.com'], ['sales@obejor.com', 'sales@obejor.com'], ['507c10c65fb14333bfa540a7cfb4b543@o417096.ingest.sentry.io'], ['hello@paykobo.com'], ['customercare@indiamart.com']]

现有实现代码 #

# 打开谷歌浏览器
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.keys import Keys

chrome_options = Options()
chrome_options.headless = False
chrome_options.add_argument("start-maximized")
# options.add_experimental_option("detach", True)
chrome_options.add_argument("--no-sandbox")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--disable-blink-features=AutomationControlled')

driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=chrome_options)
driver.get('https://www.google.com/')

# 输入搜索关键词
# var_inp=input('请输入要搜索的关键词:')
var_inp='dermalog lf10'
# 执行搜索
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.NAME, "q"))).send_keys(var_inp+Keys.RETURN)
# 获取搜索结果中的链接
res_lst=[]
res=WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.TAG_NAME,'cite')))
print(len(res))
for r in res:
    res_lst.append(driver.execute_script("return arguments[0].firstChild.textContent;", r))

print(res_lst)

# 提取每个链接页面的邮箱地址
import re
emails_lst=[]
for i in range(len(res_lst)):
    driver.get(res_lst[i])
    email_pattern = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,4}"
    html = driver.page_source
    emails = re.findall(email_pattern, html)
    driver.implicitly_wait(5)
    print(emails)
    emails_lst.append(emails)

print(emails_lst)
no_duplicates=[x for n, x in enumerate(emails_lst) if x not in emails_lst[:n]]
print(no_duplicates)
driver.close()
# 后续可添加发送邮件逻辑

解决方案：将嵌套列表转为单一无重复邮箱列表 #

替换原代码中no_duplicates的部分，用以下方式实现需求：

方法1：嵌套循环扁平化+有序去重

# 扁平化嵌套列表
flat_emails = []
for sublist in emails_lst:
    for email in sublist:
        flat_emails.append(email)
# 去重并保留邮箱出现的原始顺序（Python 3.7+）
unique_emails = list(dict.fromkeys(flat_emails))
print(unique_emails)

方法2：列表推导式扁平化+去重

# 用列表推导式快速扁平化
flat_emails = [email for sublist in emails_lst for email in sublist]
# 去重（不保证顺序）
unique_emails = list(set(flat_emails))
# 若需保留顺序，使用dict.fromkeys
unique_emails = list(dict.fromkeys(flat_emails))
print(unique_emails)

内容的提问来源于stack exchange，提问作者xlmaster