Google新闻爬虫异常：修改后仅生成空Excel文件求助

你的爬虫生成空Excel的核心原因是元素定位失败，加上URL构造、XPath语法、分页逻辑等多处错误，导致没有抓取到任何新闻数据。以下是具体问题和修复方案：

1. URL构造错误 #

原URL存在HTML转义字符&、多余符号]，且未添加日期筛选参数，无法正确定位到昨天的新闻。

修复后URL：

# 替换原news_url行
yesterday_format = (datetime.today() - timedelta(1)).strftime("%m/%d/%Y")  # Google日期格式要求MM/DD/YYYY
news_url = f'https://www.google.com/search?q={query}&tbm=nws&tbs=cdr:1,cd_min:{yesterday_format},cd_max:{yesterday_format}'

2. XPath语法与元素定位错误 #

原代码的XPath存在语法错误（缺少闭合括号），且依赖动态属性data-hveid="CBAQAA"，这类属性会随页面刷新变化，无法稳定定位。

修复后的元素定位逻辑：

# 替换原while循环内的table、li_list、area_list代码
result_container = browser.find_element(By.XPATH, '//div[@id="rso"]')  # Google搜索结果的稳定容器
news_items = result_container.find_elements(By.XPATH, './/div[@class="SoaBEf"]')  # 单条新闻的容器

3. 新闻标题与链接获取逻辑错误 #

原代码错误地从div[@role="heading"]获取链接，实际上Google新闻的链接在包裹标题的<a>标签中，标题文本则在<h3>标签内。

修复后的条目抓取逻辑：

# 替换原for a in area_list循环
for item in news_items[:min(len(news_items), news_num - idx + 1)]:
    try:
        title_elem = item.find_element(By.XPATH, './/h3')
        title = title_elem.text
        # 从标题的父级a标签获取链接
        link_elem = title_elem.find_element(By.XPATH, './..')
        n_url = link_elem.get_attribute('href')
        # 缩略图获取
        img = " "
        try:
            img_elem = item.find_element(By.XPATH, './/img[@class="tvs3Id QwxBBf"]')
            img = img_elem.get_attribute('src')
        except:
            pass
        
        news_dict[idx] = {
            'Title': title,
            'url': n_url,
            'thumbnail': img
        }
        idx += 1
    except Exception as e:
        print(f"抓取单条新闻失败: {e}")
        continue

4. 分页逻辑混乱 #

原代码同时执行next_btn.click()和browser.get(next_page_url)，导致重复跳转；且拼写错误aria-lable（应为aria-label），分页逻辑完全失效。

修复后的分页逻辑：

# 替换原try-except分页代码
try:
    next_btn = browser.find_element(By.XPATH, '//a[@id="pnnext"]')
    # 检查分页按钮是否可用（判断是否到最后一页）
    if not next_btn.is_enabled():
        break
    next_btn.click()
    cur_page += 1
    time.sleep(sleep_sec + 0.5)  # 延长等待时间确保页面加载完成
except Exception as e:
    print(f"分页失败或已到达最后一页: {e}")
    print('\n브라우저를 종료합니다.\n' + '=' * 100)
    time.sleep(0.7)
    browser.close()
    break

5. 其他优化建议 #

• 移除无用的wb = Workbook()，pandas的to_excel已足够生成Excel文件
• 添加显式等待替代固定sleep，提升稳定性：

  # 在导入部分添加
  from selenium.webdriver.support.ui import WebDriverWait
  from selenium.webdriver.support import expected_conditions as EC
  
  # 替换原browser.get(news_url)后的time.sleep(sleep_sec)
  WebDriverWait(browser, 10).until(
      EC.presence_of_element_located((By.XPATH, '//div[@id="rso"]'))
  )
  

完整修复后的代码 #

# 뉴스 크롤링.py

#######################################'사용후핵연료' 키워드 검색##################################################
import sys, os
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from datetime import datetime, timedelta
from pandas import DataFrame
import time

sleep_sec = 0.5

# User-Agent를 입력해주세요.
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'}

query = 'spent nuclear fuel'
yesterday = (datetime.today() - timedelta(1)).strftime("%Y.%m.%d")
yesterday_format = (datetime.today() - timedelta(1)).strftime("%m/%d/%Y")  # Google日期格式要求

def news_crawling():
    
    service = Service(executable_path=ChromeDriverManager().install())
    browser = webdriver.Chrome(service=service)

    print('브라우저를 실행시킵니다(자동 제어)\n')

    news_url = f'https://www.google.com/search?q={query}&tbm=nws&tbs=cdr:1,cd_min:{yesterday_format},cd_max:{yesterday_format}'
    browser.get(news_url)
    
    # 显式等待结果容器加载
    WebDriverWait(browser, 10).until(
        EC.presence_of_element_located((By.XPATH, '//div[@id="rso"]'))
    )

    print('\n크롤링을 시작합니다.')

    #####동적 제어로 페이지 넘어가며 크롤링
    news_dict = {}
    idx = 1
    cur_page = 1
    news_num = 1000000

    while True:
        result_container = browser.find_element(By.XPATH, '//div[@id="rso"]')
        news_items = result_container.find_elements(By.XPATH, './/div[@class="SoaBEf"]')
               
        for item in news_items[:min(len(news_items), news_num - idx + 1)]:
            try:
                title_elem = item.find_element(By.XPATH, './/h3')
                title = title_elem.text
                link_elem = title_elem.find_element(By.XPATH, './..')
                n_url = link_elem.get_attribute('href')
                
                img = " "
                try:
                    img_elem = item.find_element(By.XPATH, './/img[@class="tvs3Id QwxBBf"]')
                    img = img_elem.get_attribute('src')
                except:
                    pass
                    
                news_dict[idx] = {
                    'Title' : title,
                    'url' : n_url,
                    'thumbnail': img
                }
                
                idx += 1
            except Exception as e:
                print(f"抓取单条新闻失败: {e}")
                continue
            
        try:
            next_btn = browser.find_element(By.XPATH, '//a[@id="pnnext"]')
            if not next_btn.is_enabled():
                break
            next_btn.click()
            cur_page +=1 
            time.sleep(sleep_sec + 0.5)
            
        except Exception as e:
            print(f"分页失败或已到达最后一页: {e}")
            print('\n브라우저를 종료합니다.\n' + '=' * 100)
            time.sleep(0.7)
            browser.close()
            break

# 엑셀파일 추출        
    print('데이터프레임 변환\n')
    
    news_df = DataFrame(news_dict).T

    folder_path = os.getcwd()
    xlsx_file_name = '{}_{}.xlsx'.format(query, yesterday)

    news_df.to_excel(xlsx_file_name, index=False)

    print('엑셀 저장 완료 | 경로 : {}\\{}\\n'.format(folder_path, xlsx_file_name))
    
news_crawling()

内容的提问来源于stack exchange，提问作者PARK JAEHWAN