JavaScript变量大小限制解决方案？Tampermonkey爬取大数据卡顿

问题概述 #

• 使用Tampermonkey+JavaScript爬取网页数据，导出CSV后导入Google Spreadsheet做后续计算
• 数据量约270行时脚本运行正常；数据量超2000行且新增「稀有度链接」变量后，脚本卡在约1065行循环，无输出且无法完成执行
• 已尝试：将HTTP链接替换为文本、更换Firefox/Brave浏览器及新版Tampermonkey、调整浏览器缓存配置，均未解决

现有脚本核心逻辑 #

1. 页面加载后检测数据条目数量
2. 遍历所有数据，将内容累加至字符串变量
3. 遍历完成后将变量处理为CSV文件下载

问题代码 #

(function() {
'use strict';
 // Move BOTH: 0-2500 || 1-270
let piecetabsel = 0; // 0 = zone of +2000 rows Status: PROBLEM!   ||   1 = Zone of >280 Rows Status:OK
let totalpieces = 2500;  // 2500rows/line   ||   270rows/lines

let piecenamea = '';
let piecebonusa = '';
let piecepowera = '';
let themnrimage = '';
let pieceDumpFile = '';
let piecerarity = '';

var piecedumper = function(totalpieces){
    var countpieces = totalpieces - 1;
    var intra3 = setInterval(function(){
        for (koko; countpieces >= koko; koko++) {
            console.log("ID:" + koko);
            // >280 Pieces zone
            if (piecetabsel == 1){
                piecenamea = document.getElementsByClassName('piece-name')[koko].innerHTML;
                piecebonusa = document.getElementsByClassName('extra-value')[koko].innerHTML;
                piecepowera = document.getElementsByClassName('the-power')[koko].innerHTML;
                themnrimage = document.getElementsByClassName('photo-link')[koko].currentSrc;
                if(pieceDumpFile === ""){ pieceDumpFile = 'Common;' + piecenamea + ';' + piecebonusa + ';' + piecepowera + ';' + themnrimage + '\n'; }
                if(pieceDumpFile !== ""){ pieceDumpFile = pieceDumpFile + 'Common;' + piecenamea + ';' + piecebonusa + ';' + piecepowera + ';' + themnrimage + '\n'; }
            }
            // +2000 pieces zone - STUCK ZONE one 1065
            if (piecetabsel == 0){
                piecenamea = document.getElementsByClassName('collection-product-name')[koko].innerHTML;
                piecebonusa = document.getElementsByClassName('collection-bonus-amount')[koko].innerHTML.replace(/[+ ]/g, '').replace(".", ",");
                piecepowera = document.getElementsByClassName('power-value')[koko].innerHTML.replace(".", "");
                piecerarity = document.getElementsByClassName('collection-product-level-img-size-2')[koko].currentSrc;
                //
                // same has 270pieces zone ... BUT. with my workaround to decrease var size. NO SUCCESS
                if (piecerarity.includes('lv_2')) { pieceraritya = 'Uncommon'; }
                else if (piecerarity.includes('lv_3')) { pieceraritya = 'Rare'; }
                else if (piecerarity.includes('lv_4')) { pieceraritya = 'Epic'; }
                else if (piecerarity.includes('lv_5')) { pieceraritya = 'Legendary'; }
                else if (piecerarity.includes('lv_6')) { pieceraritya = 'mythic'; }
                else if (piecerarity.includes('lv_7')) { pieceraritya = 'SpecialOnes'; }
                else { pieceraritya = 'Common'; }
                if(pieceDumpFile === ""){ pieceDumpFile = pieceraritya + ';' + piecenamea + ';' + piecebonusa + ';' + piecepowera + '\n'; }
                if(pieceDumpFile !== ""){ pieceDumpFile = pieceDumpFile + pieceraritya + ';' + piecenamea + ';' + piecebonusa + ';' + piecepowera + '\n'; }
            }
        }
        clearInterval(intra3);
        var allDump='Rarity;Name;Power;Extra;Photo\n'+pieceDumpFile;
        downloadZone(allDump);
    }, 9000);
}
var downloadZone = function(allDump){
    var data=allDump;
    var a = document.createElement("a");
    a.href = 'data:application/csv;charset=utf-8,' + encodeURIComponent(data);
    a.download = 'MyInfoScrapper.csv';
    console.log("Generate CSV File and Downloading");
    a.click();
}
piecedumper(totalpieces);
})();

核心问题分析 #

1. 未声明变量导致中断：pieceraritya未提前声明，在严格模式下会抛出ReferenceError，直接终止脚本执行，这很可能是卡在1065行的直接原因
2. DOM重复查询性能损耗：循环内反复调用document.getElementsByClassName，每次都会重新遍历DOM树，大循环下严重拖慢执行速度
3. 字符串拼接内存开销：用字符串累加方式构建CSV内容，JavaScript字符串是不可变类型，每次拼接都会生成新字符串，2000+行数据会导致内存占用急剧上升，触发浏览器性能限制
4. 长阻塞主线程：一次性遍历2000+行的同步循环会完全阻塞浏览器主线程，导致页面无响应、脚本卡顿

解决方案 #

1. 修复变量未定义问题 #

在变量声明区域添加let pieceraritya = '';，避免严格模式下的报错：

let piecenamea = '';
let piecebonusa = '';
let piecepowera = '';
let themnrimage = '';
let pieceDumpFile = '';
let piecerarity = '';
let pieceraritya = ''; // 新增这行

2. 预加载DOM元素，减少重复查询 #

提前将需要的DOM元素转为数组，循环时直接从数组取值，避免重复遍历DOM：

var piecedumper = function(totalpieces){
    var countpieces = totalpieces - 1;
    // 预加载所有需要的DOM元素
    const productNames = Array.from(document.getElementsByClassName('collection-product-name'));
    const bonusAmounts = Array.from(document.getElementsByClassName('collection-bonus-amount'));
    const powerValues = Array.from(document.getElementsByClassName('power-value'));
    const rarityImgs = Array.from(document.getElementsByClassName('collection-product-level-img-size-2'));
    
    var intra3 = setInterval(function(){
        for (koko; countpieces >= koko; koko++) {
            console.log("ID:" + koko);
            if (piecetabsel == 0){
                // 直接从数组取元素，避免重复查询DOM
                piecenamea = productNames[koko].innerHTML;
                piecebonusa = bonusAmounts[koko].innerHTML.replace(/[+ ]/g, '').replace(".", ",");
                piecepowera = powerValues[koko].innerHTML.replace(".", "");
                piecerarity = rarityImgs[koko].currentSrc;
                // ... 其余逻辑不变
            }
        }
        // ... 其余逻辑不变
    }, 9000);
}

3. 用数组代替字符串累加，优化内存 #

将pieceDumpFile改为数组，每一行数据push到数组中，最后用join('\n')拼接，避免大量字符串复制开销：

// 初始化改为数组
let pieceDumpFile = [];

// 循环中替换字符串拼接逻辑
if (piecetabsel == 0){
    // ... 前面取值逻辑不变
    const row = `${pieceraritya};${piecenamea};${piecebonusa};${piecepowera}`;
    pieceDumpFile.push(row);
}

// 最后拼接成CSV内容
var allDump = 'Rarity;Name;Power;Extra;Photo\n' + pieceDumpFile.join('\n');

4. 拆分循环，避免主线程阻塞 #

将大循环拆分为多个小批次，用setTimeout让浏览器有时间处理其他任务，避免卡顿：

var piecedumper = function(totalpieces){
    const batchSize = 500; // 每批处理500行
    let currentIndex = 0;
    const productNames = Array.from(document.getElementsByClassName('collection-product-name'));
    // ... 预加载其他DOM数组
    let pieceDumpFile = [];
    
    const processBatch = () => {
        const endIndex = Math.min(currentIndex + batchSize, totalpieces);
        for (; currentIndex < endIndex; currentIndex++) {
            console.log("ID:" + currentIndex);
            // ... 处理单条数据的逻辑，将行push到pieceDumpFile数组
            piecenamea = productNames[currentIndex].innerHTML;
            piecebonusa = bonusAmounts[currentIndex].innerHTML.replace(/[+ ]/g, '').replace(".", ",");
            piecepowera = powerValues[currentIndex].innerHTML.replace(".", "");
            piecerarity = rarityImgs[currentIndex].currentSrc;
            
            if (piecerarity.includes('lv_2')) { pieceraritya = 'Uncommon'; }
            else if (piecerarity.includes('lv_3')) { pieceraritya = 'Rare'; }
            else if (piecerarity.includes('lv_4')) { pieceraritya = 'Epic'; }
            else if (piecerarity.includes('lv_5')) { pieceraritya = 'Legendary'; }
            else if (piecerarity.includes('lv_6')) { pieceraritya = 'mythic'; }
            else if (piecerarity.includes('lv_7')) { pieceraritya = 'SpecialOnes'; }
            else { pieceraritya = 'Common'; }
            
            const row = `${pieceraritya};${piecenamea};${piecebonusa};${piecepowera}`;
            pieceDumpFile.push(row);
        }
        
        if (currentIndex < totalpieces) {
            // 还有未处理的数据，延迟下一批处理
            setTimeout(processBatch, 50);
        } else {
            // 所有数据处理完成，导出CSV
            const allDump = 'Rarity;Name;Power;Extra;Photo\n' + pieceDumpFile.join('\n');
            downloadZone(allDump);
        }
    };
    
    processBatch(); // 启动分批处理
}

5. 分文件导出实现 #

如果单文件过大，可设置每N行导出一个独立文件，示例如下：

// 新增分批次导出逻辑
const batchSize = 1000; // 每1000行导出一个文件
let batchCount = 1;
let currentBatch = [];
let currentIndex = 0;

const processBatch = () => {
    const endIndex = Math.min(currentIndex + batchSize, totalpieces);
    const productNames = Array.from(document.getElementsByClassName('collection-product-name'));
    const bonusAmounts = Array.from(document.getElementsByClassName('collection-bonus-amount'));
    const powerValues = Array.from(document.getElementsByClassName('power-value'));
    const rarityImgs = Array.from(document.getElementsByClassName('collection-product-level-img-size-2'));
    
    for (; currentIndex < endIndex; currentIndex++) {
        console.log("ID:" + currentIndex);
        // ... 处理单条数据逻辑
        piecenamea = productNames[currentIndex].innerHTML;
        piecebonusa = bonusAmounts[currentIndex].innerHTML.replace(/[+ ]/g, '').replace(".", ",");
        piecepowera = powerValues[currentIndex].innerHTML.replace(".", "");
        piecerarity = rarityImgs[currentIndex].currentSrc;
        
        if (piecerarity.includes('lv_2')) { pieceraritya = 'Uncommon'; }
        else if (piecerarity.includes('lv_3')) { pieceraritya = 'Rare'; }
        else if (piecerarity.includes('lv_4')) { pieceraritya = 'Epic'; }
        else if (piecerarity.includes('lv_5')) { pieceraritya = 'Legendary'; }
        else if (piecerarity.includes('lv_6')) { pieceraritya = 'mythic'; }
        else if (piecerarity.includes('lv_7')) { pieceraritya = 'SpecialOnes'; }
        else { pieceraritya = 'Common'; }
        
        const row = `${pieceraritya};${piecenamea};${piecebonusa};${piecepowera}`;
        currentBatch.push(row);
    }
    
    if (currentIndex < totalpieces) {
        // 导出当前批次
        const batchContent = 'Rarity;Name;Power;Extra;Photo\n' + currentBatch.join('\n');
        downloadZone(batchContent, `MyInfoScrapper-Part${batchCount}.csv`);
        currentBatch = [];
        batchCount++;
        setTimeout(processBatch, 50);
    } else {
        // 导出剩余数据
        if (currentBatch.length > 0) {
            const batchContent = 'Rarity;Name;Power;Extra;Photo\n' + currentBatch.join('\n');
            downloadZone(batchContent, `MyInfoScrapper-Part${batchCount}.csv`);
        }
        console.log("所有文件导出完成");
    }
};

// 修改downloadZone函数支持自定义文件名
var downloadZone = function(allDump, filename = 'MyInfoScrapper.csv'){
    var data=allDump;
    var a = document.createElement("a");
    a.href = 'data:application/csv;charset=utf-8,' + encodeURIComponent(data);
    a.download = filename;
    console.log(`Generate CSV File ${filename} and Downloading`);
    a.click();
}

// 启动处理
processBatch();

内容的提问来源于stack exchange，提问作者hellresistor