如何在循环中正确追加JSON对象？当前生成的JSON格式有误

错误现象 #

使用appendFile写入数据后，生成的JSON是多个独立对象拼接的无效格式，示例：

{"BUSINESS":[{"title":"Luling City Market"},{"image":"https://i1.ypcdn.com/blob/f714c6a156c8bb5fb93178c0b1c4bc038dd96ec9_150x150_crop.jpg"}]}{"BUSINESS":[{"title":"Barry's Pizza"},{"image":"https://i1.ypcdn.com/blob/3b90e6b5687f5edf98e1abf1e9f74ddaf8f90e1d_150x150_crop.jpg"}]}{"BUSINESS":[{"title":"Luling City Market"},{"image":"https://i2.ypcdn.com/blob/f714c6a156c8bb5fb93178c0b1c4bc038dd96ec9_150x150_crop.jpg"}]}

用户原代码 #

const puppeteer = require('puppeteer');
var fs = require('fs');

async function scrapeProduct(url){
const file = fs.readFileSync('restaurants.json')

var scrapeTitle = '//*[@id="main-header"]/article/div/h1';
var scrapeImage = '//*[@id="main-header"]/article/a/img';

const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto(url);

const [el] = await page.$x(scrapeImage);
const src = await el.getProperty('src');
const image = await src.jsonValue();

const [el2] = await page.$x(scrapeTitle);
const txt = await el2.getProperty('textContent');
const title = await txt.jsonValue();

console.log({image, title});
var product = {
    'BUSINESS_TITLE' : [title],
    'BUSINESS_IMAGE' : [image]
}

if (file.length == 0){

    fs.writeFile('restaurants.json', JSON.stringify(product), (err) =>{
        if (err) { console.error(err); return; };
        console.log('Created File');
    })
}else{

    fs.appendFile('restaurants.json', JSON.stringify(product), (err) =>{
        if (err) { console.error(err); return; };
        console.log('Updated File');
    })
}

browser.close();
}

var urls = [
'https://www.yellowpages.com/houston-tx/mip/luling-city-market-1615603?lid=1000597596315',
'https://www.yellowpages.com/houston-tx/mip/barrys-pizza-472089383?lid=1001593326336'
]

for(var val in urls){
    scrapeProduct(urls[val]);
}

期望的正确格式 #

{"BUSINESS-TITLE":["1","2"],"BUSINESS-IMAGE":["1","2"]}

核心问题是直接追加JSON字符串会破坏JSON结构，且原代码循环调用异步函数存在并发读写冲突。下面提供两种可行方案：

方案1：一次性收集所有数据后写入（推荐） #

这种方式避免频繁文件IO，同时消除并发冲突：

const puppeteer = require('puppeteer');
const fs = require('fs').promises; // 使用Promise版本的fs，适配async/await

// 单个页面抓取逻辑
async function scrapeSingleProduct(url) {
    const scrapeTitle = '//*[@id="main-header"]/article/div/h1';
    const scrapeImage = '//*[@id="main-header"]/article/a/img';

    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto(url);

    // 抓取图片地址
    const [el] = await page.$x(scrapeImage);
    const src = await el.getProperty('src');
    const image = await src.jsonValue();

    // 抓取商户名称
    const [el2] = await page.$x(scrapeTitle);
    const txt = await el2.getProperty('textContent');
    const title = await txt.jsonValue();

    await browser.close(); // 确保浏览器正常关闭
    return { title, image };
}

// 批量抓取并写入文件的主函数
async function scrapeAndSave() {
    const urls = [
        'https://www.yellowpages.com/houston-tx/mip/luling-city-market-1615603?lid=1000597596315',
        'https://www.yellowpages.com/houston-tx/mip/barrys-pizza-472089383?lid=1001593326336'
    ];

    // 批量处理所有URL的抓取请求
    const results = await Promise.all(urls.map(url => scrapeSingleProduct(url)));

    // 整理为期望的JSON结构
    const finalData = {
        "BUSINESS-TITLE": results.map(item => item.title),
        "BUSINESS-IMAGE": results.map(item => item.image)
    };

    // 写入文件，添加格式化参数提升可读性
    try {
        await fs.writeFile('restaurants.json', JSON.stringify(finalData, null, 2));
        console.log('数据已成功写入restaurants.json');
    } catch (err) {
        console.error('写入文件失败:', err);
    }
}

// 执行主函数
scrapeAndSave();

方案2：增量追加数据（适合多次运行脚本的场景） #

如果需要每次运行脚本都往现有文件追加新数据，需先读取并解析现有JSON，合并后再写入：

const puppeteer = require('puppeteer');
const fs = require('fs').promises;

async function scrapeSingleProduct(url) {
    const scrapeTitle = '//*[@id="main-header"]/article/div/h1';
    const scrapeImage = '//*[@id="main-header"]/article/a/img';

    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto(url);

    const [el] = await page.$x(scrapeImage);
    const src = await el.getProperty('src');
    const image = await src.jsonValue();

    const [el2] = await page.$x(scrapeTitle);
    const txt = await el2.getProperty('textContent');
    const title = await txt.jsonValue();

    await browser.close();
    return { title, image };
}

async function scrapeAndAppend() {
    const urls = [
        'https://www.yellowpages.com/houston-tx/mip/luling-city-market-1615603?lid=1000597596315',
        'https://www.yellowpages.com/houston-tx/mip/barrys-pizza-472089383?lid=1001593326336'
    ];

    const newResults = await Promise.all(urls.map(url => scrapeSingleProduct(url)));

    // 初始化现有数据结构，若文件不存在则使用空数组
    let existingData = { "BUSINESS-TITLE": [], "BUSINESS-IMAGE": [] };
    try {
        const fileContent = await fs.readFile('restaurants.json', 'utf8');
        existingData = JSON.parse(fileContent);
    } catch (err) {
        console.log('未找到现有文件，将创建新文件');
    }

    // 合并新抓取的数据
    existingData["BUSINESS-TITLE"].push(...newResults.map(item => item.title));
    existingData["BUSINESS-IMAGE"].push(...newResults.map(item => item.image));

    // 写入文件
    try {
        await fs.writeFile('restaurants.json', JSON.stringify(existingData, null, 2));
        console.log('数据已成功追加到restaurants.json');
    } catch (err) {
        console.error('写入文件失败:', err);
    }
}

scrapeAndAppend();

关键改进点 #

• 使用Promise.all批量处理异步抓取，避免并发读写冲突
• 先收集所有数据再统一写入，保证JSON结构完整
• 采用fs.promises替代回调版本的fs，配合async/await简化异步逻辑
• 添加JSON格式化参数（JSON.stringify的第三个参数），提升文件可读性

内容的提问来源于stack exchange，提问作者Daniel Armstrong