Selenium爬取AIESEC机会页面时find_element()无法定位元素

爬取AIESEC志愿机会页面（示例链接：https://aiesec.org/opportunity/global-volunteer/1291411）时，尝试提取住宿、交通、餐饮等信息写入电子表格，运行代码触发selenium.common.exceptions.NoSuchElementException，提示无法定位餐饮相关元素，试过XPATH和CSS选择器均未解决。

selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"/html/body/app-root/div[1]/app-opportunity-v2/div[2]/div[2]/div[1]/div/div/section[5]/app-logistics/div/div[1]/div[3]/p[2]"}

import time
import gspread
from selenium import webdriver
from selenium.common import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By

CHROME_DRIVER_PATH = r'C:\Users\hp\PycharmProjects\AIESECSCRAPER\chromedriver_win32\chromedriver.exe'


chrome_options = Options()
chrome_options.add_argument("start-maximized")
chrome_options.add_argument('disable-infobars')
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--headless')

browser = webdriver.Chrome(executable_path=CHROME_DRIVER_PATH, options=chrome_options)
opp_links = wks.col_values(4)
MC_row = 1
project_row = 1
slots_row = 1
acc_pro = 1
acc_cov = 1
tra_pro = 1
tra_cov = 1
food_pro = 1
food_cov = 1
food_no = 1

for url in opp_links:
    #try:
        Transportation = ''
        Food = ''
        browser.get(url)
        time.sleep(20)
        MC_name = browser.find_element(By.XPATH, "//*[@id=\"info-main\"]/section/div[1]/div/span[3]").get_attribute(
            'innerHTML')
        project_name = browser.find_element(By.XPATH, "//*[@id=\"info-main\"]/section/div[1]/p[2]").get_attribute(
            'innerHTML')
        Accommodation = browser.find_element(By.XPATH,
                                             "//*[@id=\"logistics\"]/app-logistics/div/div[1]/div[1]/p[2]").get_attribute(
            'innerHTML')
        try:
            Transportation = browser.find_element(By.XPATH,
                                                  "//*[@id=\"logistics\"]/app-logistics/div/div[1]/div[2]/p[2]").get_attribute(
                'innerHTML')
        except NoSuchElementException:
            wks.update_acell(f'J{tra_pro}', 'not_provided')
            wks.update_acell(f'K{tra_cov}', 'not_covered')

        try:
            Food = browser.find_element(By.XPATH,
                                        "//*[@id=\"logistics\"]/app-logistics/div/div[1]/div[3]/p[2]").get_attribute(
                'innerHTML')
        except NoSuchElementException:
            wks.update_acell(f'L{food_pro}', 'not_provided')
            wks.update_acell(f'M{food_cov}', 'not_covered')
            wks.update_acell(f'N{food_no}', '0')

        slots = []
        try:
            for label in range(1, 30):
                date = browser.find_element(By.XPATH,
                                        f"//*[@id=\"slots\"]/app-slots/div/label[{label}]/div/p").get_attribute(
                'innerHTML')
                slots.append(date)
        except:
            pass
        print(MC_name)
        print(project_name)
        print(slots)
        print(Accommodation)
        print(Transportation)
        print(Food)

        wks.update_acell(f'A{MC_row}', MC_name)
        wks.update_acell(f'C{slots_row}', str(slots))
        wks.update_acell(f'F{project_row}', project_name)

        if Accommodation == 'A place to stay will be provided and paid for.':
            wks.update_acell(f'H{acc_pro}', 'provided')
            wks.update_acell(f'I{acc_cov}', 'covered')
        elif Accommodation == 'A place to stay will be provided but not paid for. You will have to cover the cost.':
            wks.update_acell(f'H{acc_pro}', 'provided')
            wks.update_acell(f'I{acc_cov}', 'not_covered')
        elif Transportation == "A means of transportation will be provided and paid for.":
            wks.update_acell(f'J{tra_pro}', 'provided')
            wks.update_acell(f'K{tra_cov}', 'covered')
        elif Transportation == 'A means of transportation will be provided but not paid for. You will have to cover the cost.':
            wks.update_acell(f'J{tra_pro}', 'provided')
            wks.update_acell(f'K{tra_cov}', 'not_covered')
        elif Food == '1  meals  per day will be provided and paid for.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'covered')
            wks.update_acell(f'N{food_no}', '1')
        elif Food == '2  meals  per day will be provided and paid for.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'covered')
            wks.update_acell(f'N{food_no}', '2')
        elif Food == '3  meals  per day will be provided and paid for.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'covered')
            wks.update_acell(f'N{food_no}', '3')
        elif Food == '1  meals  per day will be provided but not paid for. You will have to cover the cost.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'not_covered')
            wks.update_acell(f'N{food_no}', '1')
        elif Food == '2  meals  per day will be provided but not paid for. You will have to cover the cost.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'not_covered')
            wks.update_acell(f'N{food_no}', '2')
        elif Food == '3  meals  per day will be provided but not paid for. You will have to cover the cost.':
            wks.update_acell(f'L{food_pro}', 'provided')
            wks.update_acell(f'M{food_cov}', 'not_covered')
            wks.update_acell(f'N{food_no}', '3')

        MC_row = MC_row + 1
        project_row = project_row + 1
        slots_row = slots_row + 1
        acc_pro = acc_pro + 1
        acc_cov = acc_cov + 1
        tra_pro = tra_pro + 1
        tra_cov = tra_cov + 1
        food_pro = food_pro + 1
        food_cov = food_cov + 1
    #except:
        #print('error in this URL: ' + url)
        #MC_row = MC_row + 1
        #project_row = project_row + 1
        #slots_row = slots_row + 1
        #acc_pro = acc_pro + 1
        #acc_cov = acc_cov + 1
        #tra_pro = tra_pro + 1
        #tra_cov = tra_cov + 1
        #food_pro = food_pro + 1
        #food_cov = food_cov + 1
        #pass
browser.quit()

核心原因 #

1. 固定等待不可靠：time.sleep(20)无法适配网络波动、页面渲染延迟，元素未加载完成就触发查找。
2. 绝对XPATH脆弱：依赖页面DOM层级的绝对路径，一旦页面结构调整（比如组件顺序、嵌套层级变化），定位直接失效。
3. 动态内容未处理：AIESEC页面是单页应用，内容通过AJAX动态加载，固定等待无法保证元素就绪。

修复步骤 #

1. 替换固定等待为显式等待

用WebDriverWait等待元素加载完成，避免因加载慢导致的定位失败：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 替换time.sleep(20)
wait = WebDriverWait(browser, 30)
wait.until(EC.presence_of_element_located((By.ID, "logistics")))  # 等待物流模块加载

2. 使用稳定的相对定位

放弃绝对XPATH，改用文本标识+相对路径定位元素，适配页面结构变化：

# 定位餐饮信息的XPATH改为基于文本的相对路径
Food = wait.until(EC.presence_of_element_located(
    (By.XPATH, "//app-logistics//div[contains(text(), 'Food')]/following-sibling::p[2]")
)).get_attribute('innerHTML')

3. 完善异常处理逻辑

确保所有元素查找都被try-except包裹，且异常时同步递增行号，避免后续数据写入错位：

try:
    Food = wait.until(EC.presence_of_element_located(
        (By.XPATH, "//app-logistics//div[contains(text(), 'Food')]/following-sibling::p[2]")
    )).get_attribute('innerHTML')
except NoSuchElementException:
    wks.update_acell(f'L{food_pro}', 'not_provided')
    wks.update_acell(f'M{food_cov}', 'not_covered')
    wks.update_acell(f'N{food_no}', '0')
# 无论是否异常，都要递增行号
food_pro += 1
food_cov += 1
food_no += 1

4. 调试时关闭Headless模式（可选）

暂时注释--headless参数，手动查看页面加载状态，排查是否有弹窗、登录拦截等问题：

# chrome_options.add_argument('--headless')  # 注释该行

内容的提问来源于stack exchange，提问作者Diaa Eldin