SeleniumBase checkout bot提速求助：抢秒售商品失败

我用SeleniumBase写了个抢购Bot，目标是抢几秒就售罄的网站新品，但总是被更快的Bot抢先。目前的逻辑是每秒多次刷新页面，对比本地文件里的href链接来检测新品。试过移除time.sleep()，但删掉0.5秒的延迟后脚本直接挂起；也试过用元素等待，但网站有限制，根本没用。想知道有没有更好的新品检测方式，或者SeleniumBase是不是根本不是最优选择，求具体的提速方案。

from seleniumbase import BaseCase
import time

class MyTestClass(BaseCase):
    def save_urls_to_file(self, url_list, file_path):
        with open(file_path, 'a') as file:  # Use 'a' (append) mode instead of 'w' (write) mode
            for url in url_list:
                file.write(url + '\n')

    def read_urls_from_file(self, file_path):
        with open(file_path, 'r') as file:
            return [line.strip() for line in file]

    def get_all_href_links(self):
        elements = self.find_elements('a')
        return [element.get_attribute('href') for element in elements if element.get_attribute('href')]

    def search_for_new_urls(self, main_url, file_path, processed_file_path):
        self.open(main_url)
        all_urls = self.get_all_href_links()

        # Filter out already processed URLs
        processed_urls = self.read_urls_from_file(processed_file_path)
        new_urls = list(set(all_urls) - set(processed_urls))

        if new_urls:
            self.save_urls_to_file(new_urls, file_path)
            with open(processed_file_path, 'a') as processed_file:
                for new_url in new_urls:
                    processed_file.write(new_url + '\n')

            for new_url in new_urls:
                self.run_custom_code(new_url)

    def run_custom_code(self, url):
        self.open(url)
        self.click("div.sqs-add-to-cart-button-inner")
        self.click('div[data-test="continue-to-cart"]')
        self.click('button[data-test="cart-button"]')
        self.type('input[name="email"]', "me@gmail.com", timeout=2)
        self.click('button[data-test="continue-button"]')
        self.type('input[name="fname"]', "John")
        self.type('input[name="lname"]', "Doe")
        self.type('input[data-test="line1"]', "123 Easy")
        self.click('span:contains("Street, Any City, FL, USA")')
        time.sleep(.5)
        self.click('button[data-test="continue-button"]')
        self.click('button[data-test="continue-button"]')
        self.switch_to_frame('//iframe[contains(@name, "privateStripeFrame")]', timeout=2)
        self.find_element('input[name="cardnumber"]').send_keys('4242 4242 4242 4242')
        self.find_element('input[name="exp-date"]').send_keys('00 01')
        self.find_element('input[name="cvc"]').send_keys('346')
        self.switch_to_default_content()
        self.click('button[data-test="continue-button"]')
        self.click('button[data-test="purchase-button"]')        
        time.sleep(2)
        self.save_screenshot("new.png")

    def process_new_urls(self, main_url, file_path, processed_file_path):
        while True:
            self.search_for_new_urls(main_url, file_path, processed_file_path)
            time.sleep(.2)  

    def test_run_script(self):
        main_url = "http://example.com/shop"  
        file_path = "/home/urls.txt"  
        processed_file_path = "/home/processed_urls.txt"  

        self.process_new_urls(main_url, file_path, processed_file_path)

if __name__ == "__main__":
    MyTestClass().test_run_script()

我尝试移除time.sleep()，但0.5的延迟必须保留，否则脚本会挂起。我也尝试等待元素加载，但这不起作用，这是网站的限制。

一、优化现有SeleniumBase脚本的提速方案 #

1. 替换文件存储为内存缓存 #

当前每次检测都读写本地文件，IO操作是核心瓶颈。改成内存集合存储已处理URL，仅启动时读一次文件，后续操作全在内存中完成，文件写入异步执行：

class MyTestClass(BaseCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.processed_urls = set()

    def setup(self):
        # 启动时一次性读取已处理URL到内存
        processed_file_path = "/home/processed_urls.txt"
        self.processed_urls = set(self.read_urls_from_file(processed_file_path))

    def save_urls_async(self, new_urls, file_path, processed_file_path):
        self.save_urls_to_file(new_urls, file_path)
        with open(processed_file_path, 'a') as processed_file:
            for url in new_urls:
                processed_file.write(url + '\n')

    def search_for_new_urls(self, main_url, file_path, processed_file_path):
        self.open(main_url)
        all_urls = self.get_all_href_links()
        new_urls = list(set(all_urls) - self.processed_urls)

        if new_urls:
            self.processed_urls.update(new_urls)
            # 异步写入文件，不阻塞检测逻辑
            import threading
            threading.Thread(target=self.save_urls_async, args=(new_urls, file_path, processed_file_path)).start()
            for url in new_urls:
                self.run_custom_code(url)

2. 启用无头模式+禁用非必要资源 #

SeleniumBase的无头模式比有头模式快30%以上，再禁用图片、CSS等非必要资源，进一步缩短页面加载时间：

def test_run_script(self):
    self.open(main_url, headless=True)
    # 禁用图片、CSS加载
    self.driver.execute_cdp_cmd('Network.setBlockedURLs', {
        "urls": ["*.jpg", "*.png", "*.gif", "*.css"]
    })
    self.driver.execute_cdp_cmd('Network.enable', {})
    self.process_new_urls(main_url, file_path, processed_file_path)

3. 用JS操作替代强制sleep与元素等待 #

如果网站限制导致元素等待无效，直接用JS操作DOM，绕过前端渲染延迟：

# 替代time.sleep(0.5) + click操作
self.execute_script("""
    document.querySelector('span:contains("Street, Any City, FL, USA")').click();
    // 等待元素可点击后触发下一步
    setTimeout(() => {
        document.querySelector('button[data-test="continue-button"]').click();
    }, 200);
""")

二、更高效的新品检测：放弃页面刷新，改用API监听 #

页面刷新的效率极低，直接监听网站的商品列表API是最优解：

1. 用浏览器F12抓包，找到返回商品列表的XHR/JSON接口
2. 用requests轮询接口，对比商品ID/URL，无需加载完整页面

import requests
import time

def monitor_product_api(self, api_url):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
    }
    while True:
        try:
            resp = requests.get(api_url, headers=headers, timeout=0.5)
            products = resp.json()
            for p in products:
                if p['url'] not in self.processed_urls:
                    self.processed_urls.add(p['url'])
                    self.run_custom_code(p['url'])
        except:
            pass
        time.sleep(0.1)  # API轮询间隔可大幅缩短

这种方式比页面刷新快10倍以上，因为只请求必要的JSON数据，无需渲染页面。

三、替代SeleniumBase的轻量方案 #

如果追求极致速度，SeleniumBase（基于Selenium）的浏览器模拟开销太大，推荐以下方案：

• Playwright：比Selenium更轻量，速度更快，支持网络拦截、自动等待，API更简洁
• Pyppeteer：Python版Puppeteer，基于Chrome DevTools协议，资源占用远低于Selenium
• 纯HTTP请求：用requests直接调用API或解析页面源码，是最快的方式，但需要处理Cookie、反爬、验证码等问题

四、其他细节优化 #

• 复用浏览器实例：不要每次检测都打开新标签页，用同一个页面刷新，减少浏览器初始化开销
• 并行处理：用多线程同时监听多个接口/页面，提高检测频率
• 提前填充表单：用JS直接注入表单数据，替代type方法：

self.execute_script("""
    document.querySelector('input[name="email"]').value = 'me@gmail.com';
    document.querySelector('input[name="fname"]').value = 'John';
    document.querySelector('input[name="lname"]').value = 'Doe';
""")

内容的提问来源于stack exchange，提问作者John