从谷歌100条搜索结果爬取邮箱的两处技术问题求助

问题一：谷歌搜索结果仅返回10个站点的原因及解决方法 #

原因 #

谷歌搜索默认仅加载第一页的10条结果，后续结果需要手动触发翻页或滚动才会渲染到页面中，直接获取cite元素只能拿到当前已加载的10个站点。

解决方法 #

模拟翻页操作，循环点击下一页按钮，直到获取到100个有效站点或无更多结果。需加入页面加载等待逻辑，避免元素未加载完成导致报错：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import validators

driver = webdriver.Chrome()
search_url = "https://www.google.com/search?q=%D0%B3%D1%80%D1%83%D0%B7%D0%BE%D0%BF%D0%B5%D1%80%D0%B5%D0%B2%D0%BE%D0%B7%D0%BA%D0%B8+%D0%B0%D1%80%D1%85%D0%B0%D0%BD%D0%B3%D0%B5%D0%BB%D1%8C%D1%81%D0%BA+%D0%B0%D1%81%D1%82%D1%80%D0%B0%D1%85%D0%B0%D0%BD%D1%8C&ei=AQhEZNPgBtSF9u8Pop-4qA4&ved=0ahUKEwiT5YuZ8b3-AhXUgv0HHaIPDuUQ4dUDCBA&uact=5&oq=%D0%B3%D1%80%D1%83%D0%B7%D0%BE%D0%BF%D0%B5%D1%80%D0%B5%D0%B2%D0%BE%D0%B7%D0%BA%D0%B8+%D0%B0%D1%80%D1%85%D0%B0%D0%BD%D0%B3%D0%B5%D0%BB%D1%8C%D1%81%D0%BA+%D0%B0%D1%81%D1%82%D1%80%D0%B0%D1%85%D0%B0%D0%BD%D1%8C&gs_lcp=Cgxnd3Mtd2l6LXNlcnAQAzIHCCEQoAEQCjIHCCEQoAEQCjoECAAQRzoFCAAQgAQ6BggAEBYQHjoFCCEQoAE6BAghEBVKBAhBGABQmAZYthhgjRxoAHACeACAAaEHiAHOHJIBDTAuMS4wLjEuMi4xLjKYAQCgAQHIAQjAAQE&sclient=gws-wiz-serp"
driver.get(search_url)

results_list = []
target_count = 100

while len(results_list) < target_count:
    # 获取当前页的站点链接
    current_cites = driver.find_elements(By.TAG_NAME, 'cite')
    for cite in current_cites:
        url = cite.text.replace(">", "/").replace("›", "/").replace(" ", "")
        if validators.url(url):
            results_list.append(url)
    # 去重，避免重复站点
    results_list = list(dict.fromkeys(results_list))
    if len(results_list) >= target_count:
        break
    # 点击下一页，处理加载等待
    try:
        next_button = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.ID, "pnnext"))
        )
        next_button.click()
        # 等待页面刷新完成
        WebDriverWait(driver, 10).until(
            EC.staleness_of(current_cites[0])
        )
    except:
        # 无更多页面，终止循环
        break

# 截取前100个有效结果
results_list = results_list[:target_count]
print(f"成功获取{len(results_list)}个有效站点")

问题二：循环爬取每个站点邮箱的正确写法 #

原代码问题 #

循环中始终获取的是谷歌搜索页面的源码，并未跳转到目标站点，因此无法提取到目标站点的邮箱。

解决方法 #

遍历站点列表，逐个用driver.get()打开目标站点，等待页面加载后提取源码，再用正则匹配邮箱。同时加入异常处理，应对站点无法访问等情况：

import re

# 复用原有的邮箱正则表达式
email_pattern = r'''(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])'''

all_emails = []

for url in results_list:
    try:
        driver.get(url)
        # 等待页面主体加载完成
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.TAG_NAME, "body"))
        )
        page_source = driver.page_source
        # 提取邮箱，忽略大小写匹配
        emails = re.findall(email_pattern, page_source, re.IGNORECASE)
        # 去重后加入总列表
        for email in emails:
            if email not in all_emails:
                all_emails.append(email)
    except Exception as e:
        print(f"处理站点{url}时出错: {str(e)}")
        continue

print(f"共提取到{len(all_emails)}个唯一邮箱")
print(all_emails)

内容的提问来源于stack exchange，提问作者Levdar Armenia