Node.js网页爬虫崩溃，求调试方案及ElementNotInteractableError解决

问题背景 #

我是JavaScript新手，首次在StackOverflow发帖。将可正常运行的Python网页爬虫移植到JavaScript后，程序运行崩溃，不清楚原因，需要调试指引。我使用Mac Sonoma系统下的Node v21.7.3环境。

代码片段 #

const axios = require('./node_modules/axios/dist/node/axios.cjs');
const { URL } = require('url');
const { Builder, Browser, By, WebDriver } = require('selenium-webdriver');

class Scraper {
    constructor() {
        this.driver = new Builder().forBrowser(Browser.CHROME).build();
        this.driver.manage().window().maximize();
        this.driver.manage().setTimeouts({implicit: 5000});
        this.filelist = [];
        this.links = [];
    }

    async scrape(element = null) {
        if (typeof element === 'string') {
            this.links.push(element);
            await this.driver.get(element);
        }
        console.log('before find elements');
        const elements = await this.driver.findElements(By.css('a'));
        console.log('after find elements');
        // console.log(elements)
        for(let i = 0; i < elements.length; i++) {
            console.log('before get href');
            try {
                let link = await elements[i].getAttribute('href');
                console.log(i);
                console.log(link);
                if (link[link.length - 1] == '/') {
                    link = link.substring(0, link.length - 1);
                }
                if (link &&
                    !(typeof element === 'string' && link === element) &&
                    !((link.includes('#') && link.split('#')[0] in this.links) ||
                    (this.links.includes(link)))) {
                    // proceed only if it is not the same page or page already visited
                    if (!link.startsWith('javascript')) {
                        // console.log(link);
                        this.links.push(link);
                        try {
                            const url = new URL(link);
                            let pathname = '';
                            if (url.hostname === 'www.supermicro.com') {
                                if (url.pathname[url.pathname.length - 1] == '/') {
                                    pathname = url.pathname.substring(0, url.pathname.length - 1);
                                } else {
                                    pathname = url.pathname;
                                }
                                const urlparts = pathname.split('/')
                                if ((urlparts[urlparts.length - 1].length > 4) &&
                                    (urlparts[urlparts.length - 1].endsWith(".pdf") ||
                                    urlparts[urlparts.length - 1].endsWith(".txt"))) {
                                    this.filelist.push(tail);
                                    await axios({
                                          url: link,
                                          method: 'GET',
                                          responseType: 'blob',
                                    }).then((response) => {
                                        const href = URL.createObjectURL(response.data);
                                           //const url2 = window.URL.createObjectURL(new Blob([response.data]));
                                        const link2 = document.createElement('a');
                                        link2.href = href;
                                        link2.setAttribute('download', urlparts[urlparts.length - 1]);
                                        document.body.appendChild(link2);
                                        link2.click();
                                        document.body.removeChild(link2);
                                        URL.revokeObjectURL(href);
                                        });
                                } else {
                                    const c = await elements[i].getAttribute('class');
                                    if ((c != 'visually-hidden focusable') &&
                                        elements[i].isDisplayed() &&
                                        elements[i].isEnabled()) {
                                        try {
                                            console.log('before click');
                                            elements[i].click();
                                            console.log('after click');
                                        } catch(error) {
                                            // await this.driver.get(link);
                                            console.log(error);
                                        }
                                        this.links.push(link);
                                        console.log('before scrape');
                                        await this.scrape(elements[i]);
                                    } else {
                                        this.links.push(link);
                                    }
                                }
                            }
                        } catch(error) {
                            // not a valid url
                            console.log(error);
                        }
                    }
                }
            } catch(error) {
                console.log(error);
            }
        }
    }
}

function scrape_website(url) {
    const scraper = new Scraper();
    try {
        scraper.scrape(url);
        console.log(scraper.filelist);
    } catch(error) {
        console.log(error);
        scraper.driver.close();
    }
}

const url = "https://www.supermicro.com/en";
scrape_website(url);

报错信息 #

(base) MacBook-Pro:public me$ node web_scraper.js 
[]
before find elements
after find elements
before get href


6
https://www.web_site.com/en/quicktabs/nojs/products_menu_enterprise/0
before click
after click
before scrape
before find elements
after find elements
before get href
/Users/me/dev/com.web_site.javascript.first/public/node_modules/selenium-webdriver/lib/error.js:521
    let err = new ctor(data.message)
              ^

ElementNotInteractableError: element not interactable
  (Session info: chrome=125.0.6422.141)
    at Object.throwDecodedError (/Users/me/dev/com.supermicro.javascript.first/public/node_modules/selenium-webdriver/lib/error.js:521:15)
    at parseHttpResponse (/Users/me/dev/com.supermicro.javascript.first/public/node_modules/selenium-webdriver/lib/http.js:514:13)
    at Executor.execute (/Users/me/dev/com.supermicro.javascript.first/public/node_modules/selenium-webdriver/lib/http.js:446:28)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async thenableWebDriverProxy.execute (/Users/me/dev/com.web_site.javascript.first/public/node_modules/selenium-webdriver/lib/webdriver.js:740:17) {
  remoteStacktrace: '0   chromedriver                        0x0000000102af24c8 chromedriver + 4302024\n' +
    '1   chromedriver                        0x0000000102aeae10 chromedriver + 4271632\n' +
    '2   chromedriver                        0x000000010271c000 chromedriver + 278528\n' +
    '3   chromedriver                        0x000000010275f398 chromedriver + 553880\n' +
    '4   chromedriver                        0x0000000102754e24 chromedriver + 511524\n' +
    '5   chromedriver                        0x000000010275485c chromedriver + 510044\n' +
    '6   chromedriver                        0x0000000102796c5c chromedriver + 781404\n' +
    '7   chromedriver                        0x0000000102753004 chromedriver + 503812\n' +
    '8   chromedriver                        0x00000001027539ec chromedriver + 506348\n' +
    '9   chromedriver                        0x0000000102aba510 chromedriver + 4072720\n' +
    '10  chromedriver                        0x0000000102abefbc chromedriver + 4091836\n' +
    '11  chromedriver                        0x0000000102aa1754 chromedriver + 3970900\n' +
    '12  chromedriver                        0x0000000102abf8a4 chromedriver + 4094116\n' +
    '13  chromedriver                        0x0000000102a946d4 chromedriver + 3917524\n' +
    '14  chromedriver                        0x0000000102adcb08 chromedriver + 4213512\n' +
    '15  chromedriver                        0x0000000102adcc84 chromedriver + 4213892\n' +
    '16  chromedriver                        0x0000000102aeaa08 chromedriver + 4270600\n' +
    '17  libsystem_pthread.dylib             0x000000018ca09034 _pthread_start + 136\n' +
    '18  libsystem_pthread.dylib             0x000000018ca03e3c thread_start + 8\n'
}

Node.js v21.7.3

调试方法 #

• 分步打印元素状态：在尝试点击元素前，打印元素的offsetTop、offsetLeft、clientWidth、clientHeight，确认元素是否在视口内、是否有实际尺寸
• 捕获并打印元素HTML：在异常块中添加console.log(await elements[i].getAttribute('outerHTML'))，查看出问题的元素具体结构
• 启用ChromeDriver日志：初始化Driver时配置日志，比如new Builder().forBrowser(Browser.CHROME).setChromeOptions(new chrome.Options().setLoggingPrefs({browser: 'ALL', driver: 'ALL'})).build()，查看详细交互日志
• 手动模拟流程：打开目标网站，按照爬虫的步骤依次点击链接，观察页面是否有动态加载内容、弹窗或其他干扰因素

解决ElementNotInteractableError的建议 #

• 使用显式等待：替换隐式等待，用WebDriverWait等待元素可交互，示例：

  const { WebDriverWait, until } = require('selenium-webdriver');
  await new WebDriverWait(this.driver, 10).until(until.elementIsVisible(elements[i]));
  await new WebDriverWait(this.driver, 10).until(until.elementIsEnabled(elements[i]));
  

• 滚动到元素可见：点击前执行滚动操作，确保元素进入视口：

  await this.driver.executeScript('arguments[0].scrollIntoView({block: "center", inline: "center"});', elements[i]);
  

• 避免复用 stale element：页面跳转后，重新调用findElements获取最新的元素列表，不要使用之前循环中保存的元素对象
• 检查元素遮挡：确认元素没有被导航栏、弹窗或其他元素覆盖，可通过getBoundingClientRect()获取元素位置，对比其他元素的位置
• 改用JavaScript点击：如果常规点击失败，用JS模拟点击绕过交互限制：

  await this.driver.executeScript('arguments[0].click();', elements[i]);
  

内容的提问来源于stack exchange，提问作者Karen L