Selenium爬取Meta广告库时出现'NoneType'无startswith属性错误

错误原因 #

'NoneType' object has no attribute 'startswith'错误的本质是：当前选中的元素不包含aria-controls属性，调用get_attribute('aria-controls')返回None，而None没有startswith方法，因此触发报错。

修复方案与优化代码 #

1. 安全处理属性判断 #

先获取aria-controls属性值，判断其不为空后再检查前缀，避免None调用方法：

# 替换原有的判断逻辑
aria_controls = current_element.get_attribute('aria-controls')
if aria_controls and aria_controls.startswith("js_"):
    print("New ad - Traversing...")
    new_ad += 1
else:
    print("still traversing...")

2. 替换不稳定的TAB遍历方式 #

用TAB切换元素的方式极易定位到非广告元素，建议直接通过广告卡片的特征批量定位：

# 替换原有的tab遍历循环部分
last_height = browser.execute_script("return document.body.scrollHeight")
while True:
    # 定位当前页面所有广告卡片
    ads = WebDriverWait(browser, 10).until(EC.presence_of_all_elements_located(
        (By.XPATH, "//div[contains(@class, 'x1n2onr6') and @role='article']")
    ))
    
    for ad in ads:
        try:
            # 查找广告内的CTA按钮
            cta_buttons = ad.find_elements(By.XPATH, ".//div[@role='button' and @aria-busy='false' and @tabindex='0']")
            for button in cta_buttons:
                button_text = button.text.strip()
                if button_text in meta_cta_buttons:
                    # 向上遍历父节点直到找到a标签
                    parent_element = button
                    while parent_element.tag_name != 'a':
                        parent_element = parent_element.find_element(By.XPATH, "..")
                    cta_url = parent_element.get_attribute('href')
                    unique_store_urls.add(cta_url)
                    ads_traversed += 1
                    print(f"Ad traversed! Going next...")
        except NoSuchElementException:
            continue
    
    # 处理无限滚动
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)
    new_height = browser.execute_script("return document.body.scrollHeight")
    
    # 检查是否到达页脚或无新内容
    try:
        WebDriverWait(browser, 5).until(EC.presence_of_element_located(
            (By.XPATH, "//a[contains(text(),'Ad Library API')]")
        ))
        print("到达页面底部，停止爬取")
        break
    except TimeoutException:
        pass
    
    if new_height == last_height:
        print("无新内容加载，停止爬取")
        break
    last_height = new_height

3. 完整修复后的代码 #

import time
from datetime import datetime, timedelta
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException

# 爬取Meta广告库CTA链接的函数
def get_facebook_ads():

    past_date = (datetime.now() - timedelta(days=3)).strftime("%m/%d/%Y")
    meta_cta_buttons = ['Get Offer', 'Get offer', 'Open Link', 'Open link', 'Order Now', 'Order now', 'Save', 'Shop Now', 'Shop now', 'Subscribe', 'Learn More', 'Learn more', 'Contact Us', 'Contact us', 'Download']
    unique_store_urls = set()

    try:
        # 初始化浏览器并导航到目标页面
        browser = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
        browser.get("https://www.facebook.com/ads/library/?active_status=all&ad_type=all&country=ALL&q=%22%20%22&sort_data[direction]=desc&sort_data[mode]=relevancy_monthly_grouped&search_type=keyword_exact_phrase&media_type=all")
        # 处理搜索框
        search_box = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, "//input[@placeholder='Search by keyword or advertiser']")))
        search_box.click()
        search_box.clear()
        search_box.send_keys("" "" + Keys.ENTER)
        time.sleep(3)

        ads_traversed = 0
        last_height = browser.execute_script("return document.body.scrollHeight")
        
        while True:
            # 定位当前页面所有广告卡片
            ads = WebDriverWait(browser, 10).until(EC.presence_of_all_elements_located(
                (By.XPATH, "//div[contains(@class, 'x1n2onr6') and @role='article']")
            ))
            
            for ad in ads:
                try:
                    # 查找广告内的CTA按钮
                    cta_buttons = ad.find_elements(By.XPATH, ".//div[@role='button' and @aria-busy='false' and @tabindex='0']")
                    for button in cta_buttons:
                        button_text = button.text.strip()
                        if button_text in meta_cta_buttons:
                            # 向上遍历父节点直到找到a标签
                            parent_element = button
                            while parent_element.tag_name != 'a':
                                parent_element = parent_element.find_element(By.XPATH, "..")
                            cta_url = parent_element.get_attribute('href')
                            unique_store_urls.add(cta_url)
                            ads_traversed += 1
                            print(f"Ad traversed! Going next...")
                except NoSuchElementException:
                    continue
            
            # 处理无限滚动
            browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(3)
            new_height = browser.execute_script("return document.body.scrollHeight")
            
            # 检查是否到达页脚或无新内容
            try:
                WebDriverWait(browser, 5).until(EC.presence_of_element_located(
                    (By.XPATH, "//a[contains(text(),'Ad Library API')]")
                ))
                print("到达页面底部，停止爬取")
                break
            except TimeoutException:
                pass
            
            if new_height == last_height:
                print("无新内容加载，停止爬取")
                break
            last_height = new_height

        # 打印去重后的链接列表
        print("\n")
        for index, value in enumerate(unique_store_urls, start=1):
            print(f"{index}. {value}")            
    except Exception as e:
        print(e)
        browser.quit() 
        return unique_store_urls

def main():
    get_facebook_ads()
if __name__ == "__main__":
    main()

关键优化点 #

• 增加属性非空判断，避免None调用方法报错
• 替换TAB遍历为直接定位广告卡片，提升爬取稳定性
• 优化无限滚动逻辑，通过滚动高度和页脚元素判断终止条件
• 简化CTA按钮的查找逻辑，直接基于广告卡片范围搜索

内容的提问来源于stack exchange，提问作者MacEng