如何用JavaScript快速异步批量请求API并处理429 CORS错误

问题背景 #

需要在JavaScript中快速异步向https://api.dictionaryapi.dev发起200个API请求，核心要求：

• 捕获状态码429的限流错误（伴随CORS提示），用于重试或延长等待时间
• 其他错误直接抛出
• 请求执行顺序无要求

原尝试的代码通过逐步延长等待时间避免限流，但速度过慢；若移除延迟机制，会触发以下CORS和网络错误：

Cross-Origin Request Blocked: The Same Origin Policy disallows reading the remote resource at https://api.dictionaryapi.dev/api/v2/entries/en_US/anatomically. (Reason: CORS header ‘Access-Control-Allow-Origin’ missing). Status code: 429.

TypeError: NetworkError when attempting to fetch resource.

原问题代码 #

(async () => {
    function wait(seconds) {
        return new Promise(resolve => {
            setTimeout(() => {
                resolve()
            }, seconds * 1000);
        })
    }

    class FetchError extends Error {
        constructor() {
            super();
        }
    }

    let i = 0;
    let list = [];

    const someWords = ["webisode", "gin rummy", "cruder", "decarburising", "corn-cob pipe", "viva voce", "jaunting-car", "station blacks", "fit into", "dry mounting", "misfile", "gain time", "name day", "encouraged", "spahi", "tricorns", "in seventh heaven", "clarkias", "co-equal", "jomon", "plant", "data set", "sutra", "hypnotised", "talkshows", "quilts", "mince-pie", "grassroot", "crossfit", "price relative", "green plover", "abeyance", "ferrocyanide", "anatomically", "manful", "dust devils", "squibbed", "kyu grades", "girning", "transgendered", "narcosis", "go hot and cold", "gambling dens", "cochleae", "rowing machine", "overloud", "poetize", "rattlesnake", "speeddating", "pastored", "harmonised", "be worth one's weight in gold", "non-employment", "clinch", "intelligibility", "frog orchid", "grantor", "asthenic", "bandhs", "cedilla", "subaudition", "bassinets", "inferential", "paper bank", "episternum", "gaba", "apocarpous", "carolinian", "cab sauvs", "constrain", "bell-hop", "sympto-thermal method", "dance cards", "mescaline", "cobalts", "paper-taffeta", "palmtop", "sagittal", "bazooms", "stew", "doronicums", "fish kettle", "national grids", "religo", "catchphrases", "multiprogramming", "re-taken", "called", "sheet feeder", "chirognomy", "albizzia", "straight-jacketing", "strike while the iron is hot", "odalisques", "scouser", "metabolite", "breaded", "palavers", "seceding", "hunks", "consequence", "brillo pad", "biweekly", "ultrafine", "compellingly", "inert gas", "suffragis", "dedendums", "stochastic", "overcharging", "high-and-mighty", "bordures", "farm-worker", "pesthouse", "world-beater", "meditates", "enacted", "ophthalmic optician", "bed-bugs", "molds", "geocode", "leave the door open", "retrodictions", "tiresome", "explicit", "gearlevers", "profoundly", "anti-semitism", "hemstitched", "a bit much", "bituminizing", "upbeats", "preppy", "outfalls", "schlemiels", "bench seats", "monologise", "commoditize", "disinhibited", "conservatism", "talked", "annuls", "disadvantageous", "wild men", "radiation patterns", "non-literary", "round dances", "transvaluing", "slokas", "streeties", "medical emergency", "side valve", "urophilia", "chain letter", "quick-freezes", "transliterated", "dobermann pinschers", "alkali metal", "out-vote", "cachinnated", "nurdle", "quinquenniums", "interfluve", "p45", "dazzles", "nonreturnable", "midship", "overturns", "mast", "hyperkalemia", "don't", "suntan lotions", "unsteadiest", "meissen", "apprehensive", "gastarbeiters", "squabbled", "deprecated", "soft-paste", "heaths", "cabin cruiser", "millies", "unicorns", "repeat fees", "gavels", "suck-holing", "saviour siblings", "interspacing", "paused", "catoptric", "coral snakes", "blood brother", "beast of prey", "nullities", "afterpains", "lgbtqia", "act of grace", "atory", "blocker", "pervasive"];

    let success = 0;
    const successIntervals = 20;
    await someWords.reduce(async (array, item) => {
        await array;
        try {
            if (success >= successIntervals) await wait((success / successIntervals) << 0);
            let data = await fetch("https://api.dictionaryapi.dev/api/v2/entries/en_US/" + item);
            switch (data.status) {
                case 404:
                    throw new FetchError("Error 404: Cannot fetch " + item);
                default:
                    if (data.status >= 200 && data.status < 300) break;
                    throw new FetchError("Error " + data.status + ": Cannot fetch " + item);

            }
            const wordData = await data.json();

            if (wordData === null || wordData?.title === "No Definitions Found") throw new FetchError("No definition found for " + item);
            const completeWordData = {
                word: item,
                data: wordData
            };
            list.push(completeWordData);
            i++;
            success < successIntervals * 1.5 ? success++ : success = 0;
        } catch (e) {
            if (!(e instanceof FetchError || e instanceof TypeError)) throw e;
            console.error(e);
        }
    })
})();

优化方案 #

核心思路 #

1. 并发控制：使用固定数量的并发请求（比如10个），避免瞬间发起200个请求触发限流
2. 指数退避重试：遇到429错误时，用指数增长的延迟时间重试，而非全局延迟所有请求
3. 错误区分：精准捕获429相关的网络错误（因为CORS限制下无法直接获取status，只能通过错误信息判断），其他错误直接抛出

优化代码 #

(async () => {
    const BASE_URL = "https://api.dictionaryapi.dev/api/v2/entries/en_US/";
    const CONCURRENCY_LIMIT = 10; // 并发请求数，可根据API限流调整
    const MAX_RETRIES = 5; // 最大重试次数

    // 延迟函数
    const wait = ms => new Promise(resolve => setTimeout(resolve, ms));

    // 单个单词的请求逻辑，带重试
    const fetchWord = async (word, retryCount = 0) => {
        try {
            const response = await fetch(`${BASE_URL}${encodeURIComponent(word)}`);
            
            // 处理正常响应状态
            if (response.ok) {
                const data = await response.json();
                if (data?.title === "No Definitions Found") {
                    throw new Error(`No definition found for ${word}`);
                }
                return { word, data };
            }

            // 处理429状态（如果能获取到的话）
            if (response.status === 429) {
                if (retryCount >= MAX_RETRIES) {
                    throw new Error(`Max retries reached for ${word} (429)`);
                }
                // 指数退避：2^retryCount * 1000ms
                const delay = Math.pow(2, retryCount) * 1000;
                await wait(delay);
                return fetchWord(word, retryCount + 1);
            }

            // 其他非200状态直接抛出
            throw new Error(`Error ${response.status}: Cannot fetch ${word}`);
        } catch (error) {
            // 捕获CORS导致的网络错误（对应429限流）
            if (error instanceof TypeError && error.message.includes("NetworkError")) {
                if (retryCount >= MAX_RETRIES) {
                    throw new Error(`Max retries reached for ${word} (CORS/429)`);
                }
                const delay = Math.pow(2, retryCount) * 1000;
                await wait(delay);
                return fetchWord(word, retryCount + 1);
            }
            // 其他错误直接抛出
            throw error;
        }
    };

    const someWords = ["webisode", "gin rummy", "cruder", "decarburising", "corn-cob pipe", "viva voce", "jaunting-car", "station blacks", "fit into", "dry mounting", "misfile", "gain time", "name day", "encouraged", "spahi", "tricorns", "in seventh heaven", "clarkias", "co-equal", "jomon", "plant", "data set", "sutra", "hypnotised", "talkshows", "quilts", "mince-pie", "grassroot", "crossfit", "price relative", "green plover", "abeyance", "ferrocyanide", "anatomically", "manful", "dust devils", "squibbed", "kyu grades", "girning", "transgendered", "narcosis", "go hot and cold", "gambling dens", "cochleae", "rowing machine", "overloud", "poetize", "rattlesnake", "speeddating", "pastored", "harmonised", "be worth one's weight in gold", "non-employment", "clinch", "intelligibility", "frog orchid", "grantor", "asthenic", "bandhs", "cedilla", "subaudition", "bassinets", "inferential", "paper bank", "episternum", "gaba", "apocarpous", "carolinian", "cab sauvs", "constrain", "bell-hop", "sympto-thermal method", "dance cards", "mescaline", "cobalts", "paper-taffeta", "palmtop", "sagittal", "bazooms", "stew", "doronicums", "fish kettle", "national grids", "religo", "catchphrases", "multiprogramming", "re-taken", "called", "sheet feeder", "chirognomy", "albizzia", "straight-jacketing", "strike while the iron is hot", "odalisques", "scouser", "metabolite", "breaded", "palavers", "seceding", "hunks", "consequence", "brillo pad", "biweekly", "ultrafine", "compellingly", "inert gas", "suffragis", "dedendums", "stochastic", "overcharging", "high-and-mighty", "bordures", "farm-worker", "pesthouse", "world-beater", "meditates", "enacted", "ophthalmic optician", "bed-bugs", "molds", "geocode", "leave the door open", "retrodictions", "tiresome", "explicit", "gearlevers", "profoundly", "anti-semitism", "hemstitched", "a bit much", "bituminizing", "upbeats", "preppy", "outfalls", "schlemiels", "bench seats", "monologise", "commoditize", "disinhibited", "conservatism", "talked", "annuls", "disadvantageous", "wild men", "radiation patterns", "non-literary", "round dances", "transvaluing", "slokas", "streeties", "medical emergency", "side valve", "urophilia", "chain letter", "quick-freezes", "transliterated", "dobermann pinschers", "alkali metal", "out-vote", "cachinnated", "nurdle", "quinquenniums", "interfluve", "p45", "dazzles", "nonreturnable", "midship", "overturns", "mast", "hyperkalemia", "don't", "suntan lotions", "unsteadiest", "meissen", "apprehensive", "gastarbeiters", "squabbled", "deprecated", "soft-paste", "heaths", "cabin cruiser", "millies", "unicorns", "repeat fees", "gavels", "suck-holing", "saviour siblings", "interspacing", "paused", "catoptric", "coral snakes", "blood brother", "beast of prey", "nullities", "afterpains", "lgbtqia", "act of grace", "atory", "blocker", "pervasive"];

    const results = [];
    const errors = [];

    // 分批次并发执行
    for (let i = 0; i < someWords.length; i += CONCURRENCY_LIMIT) {
        const batch = someWords.slice(i, i + CONCURRENCY_LIMIT);
        const batchPromises = batch.map(word => 
            fetchWord(word)
                .then(result => results.push(result))
                .catch(err => errors.push(err))
        );
        await Promise.all(batchPromises);
    }

    console.log("成功获取的单词数据:", results);
    console.log("请求错误:", errors);
})();

方案说明 #

• 并发控制：通过分批次执行请求，每次最多发起CONCURRENCY_LIMIT个请求，避免瞬间压垮API触发限流
• 指数退避：遇到429或CORS网络错误时，重试延迟时间按2的幂次增长（1s、2s、4s...），既保证重试成功率，又不会持续占用资源
• 精准错误处理：区分429状态（如果能获取到）和CORS导致的TypeError，仅针对这两类错误重试，其他错误直接抛出
• 编码处理：使用encodeURIComponent处理带空格或特殊字符的单词，避免请求URL格式错误

内容的提问来源于stack exchange，提问作者horsey_guy