Node.js中fs.readFileSync间歇性写入部分数组至文件的问题

背景 #

从ESPN API获取NFL赛季18周的对阵数据，需调用18次周赛程API，再从响应中提取数据生成对应18个JSON文件（week1.json至week18.json）。

问题现象 #

1. 部分JSON文件仅包含预期数组的一部分数据，比如本该有16场比赛的数组，实际只有4-6条数据。
2. 并非每次调用都能更新全部18个文件，有时仅更新前几周的文件。

问题代码的核心原因 #

// iterate 18 times
for (let i = 0; i < 18; i++) {

    let weekNumber;
    weekNumber = i + 1;
    const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
    const weekJson = JSON.parse(week);

    // empty weekJson.games array
    weekJson.games = []

    // get all items
    axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
        .then(response => {
            const schedule = [];
            // get all items from response
            const items = response.data.items
            // console.log(response.data.items)

            items.forEach(item => {
                // make get call to $ref
                axios.get(item.$ref)
                    .then(response => {
                        // get name
                        const name = response.data.name
                        // get date
                        const date = response.data.date
                        // get event id
                        const eventid = response.data.id
                        // get team ids
                        let team1 = response.data.competitions[0].competitors[0].id
                        let team2 = response.data.competitions[0].competitors[1].id

                        // create new object
                        const newObject = {
                            name: name,
                            date: date,
                            eventid: eventid,
                            team1: team1,
                            team2: team2
                        }

                        // add games for week
                        weekJson.games.push(newObject);
                        fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));


                    })
                    .catch(error => {
                        console.log(error)
                    })
            })

        }).catch(error => {
            console.log(error)
        })
}

问题1（数据不全）的原因 #

• forEach是同步循环，但内部的axios.get是异步操作，所有比赛详情请求会并发执行，而非按顺序等待完成。
• 每次请求完成就调用fs.writeFileSync写入文件，此时其他比赛请求可能还未返回，数组weekJson.games尚未收集全所有数据，导致最终文件仅保存了部分已完成请求的数据。
• 并发修改同一个weekJson.games数组可能引发数据覆盖或丢失。

问题2（文件未全部更新）的原因 #

• 外层的18周循环是同步执行的，会瞬间发起所有18个周赛程API请求，同时每个周又发起多个比赛详情请求，并发请求量过大可能触发API的速率限制，导致部分请求失败，无法生成对应文件。
• 异步请求未等待全部完成，程序可能在部分请求处理完前就结束，导致后续文件未被写入。

修复后的代码解析 #

router.get('/getschedules', (req, res) => {

async function writeGames() {

// iterate 18 times
for (let i = 0; i < 18; i++) {

    let weekNumber;
    weekNumber = i + 1;
    const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
    const weekJson = JSON.parse(week);

    // empty weekJson.games array
    weekJson.games = []

    // Add await keyword to wait for a week to be processed before going to the next one
    await axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
        .then(async (response) => { // add async to be able to use await
            const schedule = [];
            // get all items from response
            const items = response.data.items
             console.log(response.data.items)
            // Use standard loop to be able to benefit from async/await
            for (let item of items) {
                // make get call to $ref
                // wait for an item to be processed before going to the next one
                await axios.get(item.$ref)
                    .then(response => {
                        // get name
                        const name = response.data.name
                        // get date
                        const date = response.data.date
                        // get event id
                        const eventid = response.data.id
                        // get team ids
                        let team1 = response.data.competitions[0].competitors[0].id
                        let team2 = response.data.competitions[0].competitors[1].id

                        // create new object
                        const newObject = {
                            name: name,
                            date: date,
                            eventid: eventid,
                            team1: team1,
                            team2: team2
                        }

                        // add games for week
                        weekJson.games.push(newObject);
                    })
                    .catch(error => {
                        console.log(error)
                    })
            }
            // moved out of the for loop since you only need to write this once
            fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));

        }).catch(error => {
            console.log(error)
        })
}
}

writeGames();

})

关键改进点 #

1. 串行处理周数据：在外层周循环中使用await，确保处理完当前周的所有数据后，再发起下一周的API请求，避免并发请求量过大触发限制。
2. 串行处理单周内的比赛数据：用for...of替代forEach，配合await等待每个比赛详情请求完成后，再处理下一场比赛，确保所有数据都被收集到数组中。
3. 统一写入文件：将fs.writeFileSync移到单周所有比赛处理完成之后，只执行一次写入，保证文件保存的是完整的周赛程数据。

内容的提问来源于stack exchange，提问作者dentalhero