用Selenium爬取谷歌搜索结果失败：返回‘No Data Found’求助

我用Selenium向谷歌提交查询，爬取前几条搜索结果，尝试将URL、标题和描述存入字典以便后续调用，但始终无法正常运行，返回‘No Data Found’。以下是原实现代码：

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common import exceptions
from selenium.webdriver.common.by import By
import re

options = Options()
options.add_argument("--headless")

def googleSearch(query):
    # specifing browser web driver
    driver = webdriver.Chrome(options=options, executable_path='chromedriver')
    
    # search query
    search_engine = "https://www.google.com/search?q="
    query = query.replace(" ","+")
    driver.get(search_engine + query + "&start=" + "0")

    # stored data
    # which will be returned by this function
    data = {}
    
    # number of search reasult count of first page
    s_len = 5
    
    for s_block in range(s_len):

        # result block
        content_block_xpath = f'''//*[@id="yuRUbf"]/div[{s_block}]/div/div'''

        # xpaths
        xpath_url = f"""{content_block_xpath}/div[1]/a"""
        xpath_title = f"""{content_block_xpath}/div[1]/a/h3"""
        xpath_description = f"""{content_block_xpath}/div[2]/span/span"""
        
        try:
                        
            # store data collected of each s_block to block {}
            block = {}

            # find url of content
            url = driver.find_element(By.XPATH, xpath_url)
            url = url.get_attribute('href')

            links.append(url.get('href'))
            # find domain name of web having content
            pattern =  r"""(https?:\/\/)?(([a-z0-9-_]+\.)?([a-z0-9-_]+\.[a-z0-9-_]+))"""
            domain = re.search(pattern, url)[0]
            print(links)

            # find title of content
#            title = driver.find_element_by_xpath(xpath_title)
            title = driver.find_element(By.XPATH, xpath_title)
            title = title.get_attribute("innerText")
            
            # find description of content
#            description = driver.find_element_by_xpath(xpath_description)
            description = driver.find_element(By.XPATH, xpath_description)
            description = description.get_attribute("innerText")

            # save all data to block {}
            block["domain"] = domain
            block["url"] = url
            block["title"] = title
            block["description"] = description

            # save block dictionary to main dictionary
            data[f'{s_block}'] = block
        
        except exceptions.NoSuchElementException:
            continue
        
        if len(data) == 0:
            raise Exception("No data found")

    driver.close()
    return data

def getQuery():
    query = str('How to change a car tire')
    link = googleSearch(query)
    print(link)

getQuery()

问题原因及修复方案 #

1. DOM结构定位错误 #

谷歌搜索结果的DOM结构会频繁更新，原XPath依赖固定ID和索引的方式极不稳定。改用更通用的定位：搜索结果容器通常带有class="g"属性，直接定位所有此类元素再遍历。

2. 未初始化变量及错误的URL处理 #

• 代码中links列表未定义就调用append，会直接抛出异常；
• url已经是通过get_attribute('href')获取的字符串，不需要再调用url.get('href')。

3. 异常判断时机错误 #

原代码在循环内部每次都判断len(data) == 0，只要第一次循环没获取到数据就抛出异常，应该放在循环结束后统一判断。

4. 反爬规避不足 #

无头模式容易被谷歌检测，需要添加额外参数模拟正常浏览器。

5. 域名提取的鲁棒性不足 #

原正则匹配可能返回None，导致[0]索引报错，需要增加判断逻辑。

修复后的代码 #

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common import exceptions
from selenium.webdriver.common.by import By
import re

options = Options()
options.add_argument("--headless=new")  # 新版无头模式，更接近正常浏览器
options.add_argument("--disable-images")  # 禁用图片加载，提升速度
options.add_argument("--window-size=1920,1080")  # 设置窗口大小
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")

def googleSearch(query):
    driver = webdriver.Chrome(options=options)
    
    search_engine = "https://www.google.com/search?q="
    query = query.replace(" ", "+")
    driver.get(f"{search_engine}{query}&start=0")

    data = {}
    s_len = 5
    links = []  # 初始化links列表

    # 获取所有搜索结果容器
    result_blocks = driver.find_elements(By.CLASS_NAME, "g")
    # 取前s_len个结果
    target_blocks = result_blocks[:s_len]

    for idx, s_block in enumerate(target_blocks):
        try:
            block = {}

            # 定位URL
            url_elem = s_block.find_element(By.TAG_NAME, "a")
            url = url_elem.get_attribute('href')
            links.append(url)

            # 提取域名
            pattern = r"(https?:\/\/)?(([a-z0-9-_]+\.)?([a-z0-9-_]+\.[a-z0-9-_]+))"
            domain_match = re.search(pattern, url)
            domain = domain_match.group(0) if domain_match else "Unknown"

            # 定位标题
            title_elem = s_block.find_element(By.TAG_NAME, "h3")
            title = title_elem.get_attribute("innerText")

            # 定位描述
            desc_elem = s_block.find_element(By.CSS_SELECTOR, "div.VwiC3b")
            description = desc_elem.get_attribute("innerText")

            block["domain"] = domain
            block["url"] = url
            block["title"] = title
            block["description"] = description
            data[str(idx)] = block

        except exceptions.NoSuchElementException:
            continue

    if len(data) == 0:
        raise Exception("No data found")

    driver.quit()  # 用quit()代替close()，彻底关闭浏览器进程
    return data

def getQuery():
    query = 'How to change a car tire'
    try:
        result = googleSearch(query)
        print(result)
    except Exception as e:
        print(e)

getQuery()

内容的提问来源于stack exchange，提问作者Stridermn