Python+Selenium：元素不存在时跳过循环迭代的问题排查

问题描述 #

• 基于Python+Selenium爬取百度B2B网站（https://b2b.baidu.com/）
• 核心需求：输入搜索词后，若首页存在company_url元素则进入详情页抓取企业信息；若不存在则跳过当前搜索词，继续处理下一个
• 异常现象：无论设置多大的循环范围，最终仅能获取一行数据

原实现代码 #

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

import time
import pandas as pd


website = 'https://b2b.baidu.com/'
path = "C:/Users/ICT/chromedriver.exe"


driver = webdriver.Chrome(path)
driver.get(website)
driver.implicitly_wait(4)
wait = WebDriverWait(driver, 10)
driver.maximize_window()

# 包含地点和关键词的搜索词来自另一个文件的dataframe
from baidu_locations import location_key_row
from baidu_locations import location_data_col
from baidu_locations import key_data_col


for i in range(1, 6):
    
    website = []
    rep_name = []
    contact = []
    location = []
    keyword = []
    business_name = []

    # 输入地点和关键词
    enter_query = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[placeholder='我要采购…']")))
    enter_query.clear()    
    enter_query.send_keys(location_key_row[i-1])

    location_query = location_data_col[i-1]
    location.append(location_query)
    keyword_query = location_data_col[i-1]
    keyword.append(keyword_query)
        
    search_type = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "li[class='search-type']")))
    search_type.click()
# 若company_url元素不可用，回到下一个enter_query继续迭代
    try:    
        company_url = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div > div:nth-child(1) > div > div > div > div.title-container > span > span.name > a')))
        website.append(company_url.get_property('href'))
                      
        first_location = wait.until(EC.element_to_be_clickable((By.XPATH, '(//span[@class="title link"])[1]')))
        first_location.click()
        
        driver.switch_to.window(driver.window_handles[1])
        
        name = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div[class='shop-contact-warp shop-contact-vertical-warp'] div[class='top'] div span[class='show-name']")))
        business_name.append(name.text)
        #print(reps)
        
        representative = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.shop-index-new-right> div > div.top > div:nth-child(1) > div > div.text > p.sub-text")))
        rep_name.append(representative.text)
        
        phone_option = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'查看电话')]")))
        phone_option.click()
        
        popup_contact = driver.window_handles[1]
        driver.switch_to.window(popup_contact)
        
        phone_number = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'p[class="phone"]')))
        contact.append(phone_number.text)
        #print(contact_no)
        
        
        time.sleep(2)
        
        return_button = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'返回')]")))
        return_button.click()
        driver.close()
        
        driver.switch_to.window(driver.window_handles[0])
  except:
        continue    
    
    
    df = pd.DataFrame({'Location': location, 'Keyword': keyword, 'Name': business_name, 'Representative': rep_name, 'Phone': contact, 'Link': website})

问题根源分析 #

1. 数据存储列表循环内初始化：每次循环都会重新创建website、rep_name等列表，之前循环的抓取数据被清空，最终仅保留最后一次循环的结果
2. 关键词赋值错误：keyword_query = location_data_col[i-1]误用了地点数据列，应该调用key_data_col[i-1]获取关键词
3. 缺少搜索触发操作：输入搜索词后未点击搜索按钮或按下回车，可能未执行搜索就直接查找结果元素，导致逻辑异常
4. 窗口切换与关闭逻辑混乱：查看电话弹窗后的窗口切换操作有误，可能导致后续循环无法正常定位元素
5. DataFrame循环内创建：每次循环都会生成新的DataFrame，覆盖之前的结果，最终仅保留最后一行数据

修正后的代码 #

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

import time
import pandas as pd


website = 'https://b2b.baidu.com/'
path = "C:/Users/ICT/chromedriver.exe"


driver = webdriver.Chrome(path)
driver.get(website)
driver.implicitly_wait(4)
wait = WebDriverWait(driver, 10)
driver.maximize_window()

# 包含地点和关键词的搜索词来自另一个文件的dataframe
from baidu_locations import location_key_row
from baidu_locations import location_data_col
from baidu_locations import key_data_col

# 数据存储列表移到循环外，避免每次循环重置
website_list = []
rep_name_list = []
contact_list = []
location_list = []
keyword_list = []
business_name_list = []

for i in range(1, 6):
    try:
        # 输入地点和关键词
        enter_query = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[placeholder='我要采购…']")))
        enter_query.clear()    
        enter_query.send_keys(location_key_row[i-1])
        
        # 修正关键词赋值错误
        location_query = location_data_col[i-1]
        location_list.append(location_query)
        keyword_query = key_data_col[i-1]
        keyword_list.append(keyword_query)
        
        # 触发搜索（按回车）
        enter_query.send_keys(Keys.ENTER)
        
        # 等待搜索结果加载
        time.sleep(2)
        
        # 检查是否存在公司链接元素
        company_url = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div > div:nth-child(1) > div > div > div > div.title-container > span > span.name > a')))
        website_list.append(company_url.get_property('href'))
        
        # 点击进入详情页并等待新窗口打开
        company_url.click()
        wait.until(EC.number_of_windows_to_be(2))
        driver.switch_to.window(driver.window_handles[1])
        
        # 抓取企业名称
        name = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div[class='shop-contact-warp shop-contact-vertical-warp'] div[class='top'] div span[class='show-name']")))
        business_name_list.append(name.text)
        
        # 抓取联系人
        representative = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.shop-index-new-right> div > div.top > div:nth-child(1) > div > div.text > p.sub-text")))
        rep_name_list.append(representative.text)
        
        # 查看电话并抓取号码
        phone_option = wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'查看电话')]")))
        phone_option.click()
        phone_number = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'p[class="phone"]')))
        contact_list.append(phone_number.text)
        
        time.sleep(1)
        
        # 关闭详情页窗口，切回主窗口
        driver.close()
        driver.switch_to.window(driver.window_handles[0])
        
    except Exception as e:
        # 打印异常信息便于排查，同时恢复窗口状态
        print(f"处理第{i}个搜索词时出错: {str(e)}")
        if len(driver.window_handles) > 1:
            driver.close()
            driver.switch_to.window(driver.window_handles[0])
        continue    

# 循环结束后统一生成DataFrame
df = pd.DataFrame({
    'Location': location_list, 
    'Keyword': keyword_list, 
    'Name': business_name_list, 
    'Representative': rep_name_list, 
    'Phone': contact_list, 
    'Link': website_list
})
# 可选：保存数据到CSV
# df.to_csv('baidu_b2b_data.csv', index=False, encoding='utf-8-sig')

driver.quit()

修正说明 #

• 将数据存储列表移至循环外，确保每次循环的结果都能追加保存
• 修正关键词赋值错误，使用正确的key_data_col获取关键词
• 添加搜索触发操作（按回车），确保搜索请求被执行
• 优化窗口切换逻辑，等待新窗口加载完成后再切换，避免元素定位失败
• 将DataFrame创建移至循环结束后，统一整合所有抓取的数据
• 完善异常处理，出现错误时切回主窗口，避免后续循环受影响，并打印异常信息便于排查

内容的提问来源于stack exchange，提问作者Temidayo Amure