如何将Puppeteer爬虫数据存入MongoDB？连接成功但无法写入

问题根源分析 #

原代码存在以下关键问题导致无法写入MongoDB：

• 模型引用名称不匹配，且Schema拼写错误
• 变量作用域错误，无法访问爬虫循环内的contact和i
• 爬虫数据字段与MongoDB模型字段不匹配
• 手机号字段类型设置错误（Number不适合存储带格式或长位数的手机号）
• MongoDB连接逻辑错误，提前关闭连接且与爬虫逻辑脱节
• 缺少错误捕获，无法定位写入失败原因

步骤1：修正Mongoose模型文件 #

修正字段拼写、补充缺失字段、调整手机号类型：

const mongoose = require('mongoose')

// 修正Schema拼写错误，补充date字段，将phone改为String类型
const huarenjieSchema = mongoose.Schema({
    infoIndex: {
        type: Number,
        unique: true,
        required: true
    },
    who: {
        type: String
    },
    phone: {
        type: String, // 手机号用String存储，避免Number类型的长度限制和格式问题
        required: true
    },
    zone: {
        type: String
    },
    workType: {
        type: String
    },
    comment: {
        type: String
    },
    date: {
        type: String // 补充date字段，与爬虫数据对应
    }
})

module.exports = mongoose.model('Huarenjie', huarenjieSchema)

步骤2：修正爬虫代码的MongoDB写入逻辑 #

移除冗余连接调用，将MongoDB写入逻辑整合到爬虫循环内，确保作用域正确，添加错误处理：

// import
const Datastore = require('nedb')
const puppeteer = require('puppeteer');
const fs = require('fs');
const { exit } = require('process');
const commandLineArgs = require('command-line-args');

// MongoDB相关
const database = require('./config/database')
const Huarenjie = require("./model/huarenjiemodel") // 修正模型引用名称

// global variables
var totalFound = 0
var headless = true
const DB_PATH = './huarenjietest2.db'
const START_LINK = 'https://www.huarenjiewang.com/info-[pageIndex].html'
const LOCATION_SeleCTOR = 'body > div.body1000 > div.location'
const COOKIE_SELECTOR = 'body > div.fc-consent-root > div.fc-dialog-container > div.fc-dialog.fc-choice-dialog > div.fc-dialog-content > div > div.fc-header.fc-dialog-restricted-content > h1'
const NotFoundSelector = 'body > div > div.tip-bgC > div'
const phoneSelector = '#show-phone'
const workTypeSelector = 'body > div.body1000 > div.wrapper > div.information_bd > div.bd_left > div.extra_contact > div > ul > li:nth-child(4)'
const zoneSelector =  'body > div.body1000 > div.wrapper > div.information_bd > div.bd_left > div.extra_contact > div > ul > li:nth-child(1)'
const whoSelector = 'body > div.body1000 > div.wrapper > div.information_bd > div.bd_left > div.extra_contact > div > ul > li:nth-child(6) > span.faburen > a:nth-child(1)'
const infoSelector = 'body > div.body1000 > div.wrapper > div.information_bd > div.bd_left > div.view_bd > p'
const dateSeloctor = 'body > div.body1000 > div.wrapper > div.information_bd > div.bd_left > div.information_hd > div.my_yp_view_title2 > div.my_yp_view_title2_l'
const COOKIE_BUTTON = 'body > div.fc-consent-root > div.fc-dialog-container > div.fc-dialog.fc-choice-dialog > div.fc-footer-buttons-container > div.fc-footer-buttons > button.fc-button.fc-cta-consent.fc-primary-button'

// command line options
const optionDefinitions = [
  { name: 'headless', type: Boolean },
  { name: 'start', type: Number },
  { name: 'end', type: Number },
  { name: 'export', type: String },
]

const options = commandLineArgs(optionDefinitions)
const startIndex = options.start
const endIndex = options.end
const outputPath = options.export

if(!startIndex || !endIndex){
  console.log('使用命令示例: node huarenjie.js --start 12321323 --end 13321323 --export output.csv')
  exit()
}

// NeDB初始化（不需要可删除）
const dbPath = DB_PATH
const db = new Datastore({ filename: dbPath, autoload: true })
db.ensureIndex({ fieldName: 'infoIndex', unique: true }, function (err) {
  if (err) {
    console.log('ensureIndex error :', err)
  }
})

function dateyyyyMMdd(d, separate = '') {
  let date = d
  var mm = date.getMonth() + 1 // getMonth()从0开始计数
  var dd = date.getDate()
  return [date.getFullYear(), (mm > 9 ? '' : '0') + mm, (dd > 9 ? '' : '0') + dd].join(separate)
}

function delay(time) {
   return new Promise(resolve => setTimeout(resolve, time))
}

async function checkPageExists(page){
  try {
    // 优先处理cookie弹窗
    await page.waitForSelector(COOKIE_BUTTON, { timeout: 3000 }).then(() => {
      return page.$eval(COOKIE_BUTTON, form => form.click())
    }).catch(() => {})
    
    // 检查页面是否存在
    await page.waitForSelector(LOCATION_SeleCTOR, { timeout: 5000 })
    // 检查是否跳转到城市选择页
    if(page.url() === 'https://www.huarenjiewang.com/changecity.php'){
      return false
    }
    return true
  } catch (error) {
    // 检查是否为404页面
    const notFound = await page.$(NotFoundSelector)
    if(notFound){
      console.info('页面不存在')
    }
    return false
  }
}

async function search() {
  // 初始化MongoDB连接
  await database.connection()
  const browser = await puppeteer.launch({ headless: headless })
  console.log('====> Puppeteer启动')
  const page = await browser.newPage();
  await page.setViewport({ width: 1280, height: 800 })

  for(var i=startIndex;i<=endIndex;i++){
    try {
      console.info(`====== 页码 : ${i}  ========`)
      await page.goto(START_LINK.replace('[pageIndex]',i))
      const exist = await checkPageExists(page)
      
      if(exist){
        let element = await page.$(LOCATION_SeleCTOR)
        let categoryValue = await page.evaluate(el => el.textContent, element)
        
        // 合并重复逻辑，减少冗余
        if(categoryValue.includes('appartement/louer') || categoryValue.includes('maison/louer')){
          let phone = await page.$(phoneSelector)
          let phoneValue = await page.evaluate(el => el.getAttribute("data-phone"), phone)
          
          if(phoneValue){
            let workElement = await page.$(workTypeSelector)
            let workValue = await page.evaluate(el => el.textContent, workElement)
            let zoneElement = await page.$(zoneSelector)
            let zoneValue = await page.evaluate(el => el.textContent, zoneElement)
            let whoElement = await page.$(whoSelector)
            let whoValue = await page.evaluate(el => el.textContent, whoElement)
            let infoElement = await page.$(infoSelector)
            let infoValue = await page.evaluate(el => el.textContent, infoElement)
            let dateElement = await page.$(dateSeloctor)
            var dateValue = await page.evaluate(el => el.textContent, dateElement)
            var dateStr = ''
            
            var reg1 = /(\d{4}-\d{2}-\d{2})/
            const found1 = dateValue.match(reg1)
            if(found1 && found1.length > 1){
              dateStr = found1[1]
            }
            if(dateValue.includes('昨天')){
              var date = new Date()
              date.setDate(date.getDate() -1)
              dateStr = dateyyyyMMdd(date,'-')
            }
            if(dateValue.includes('前天')){
              var date = new Date()
              date.setDate(date.getDate() -2)
              dateStr = dateyyyyMMdd(date,'-')
            }
            
            const contact = {
              phone : phoneValue,
              zone: zoneValue,
              workType: workValue,
              who: whoValue,
              comment: infoValue, // 匹配模型中的comment字段
              date: dateStr || '',
              infoIndex: i
            }
            
            // 写入NeDB（不需要可删除）
            db.update({ infoIndex: i }, { $set: contact }, { upsert: true }, function (err) {
              if(err) console.error('NeDB写入错误:', err)
            });
            
            // 写入MongoDB，添加错误捕获
            try {
              await Huarenjie.findOneAndUpdate(
                { infoIndex: i }, 
                { $set: contact }, 
                { upsert: true, new: true }
              )
              console.info(`${i}#############>MongoDB保存成功`)
            } catch (mongoErr) {
              console.error(`${i}=>MongoDB写入失败:`, mongoErr)
            }
            
          } else {
            console.info(`${i}=>跳过==>手机号缺失`)
          }
        } else {
          console.info(`${i}=>跳过==>分类不符合`)
        }
      }
    } catch (error) {
      console.info(`${i}=>页面读取错误:`, error)
    }
  }
  
  // 爬虫结束后关闭资源
  await browser.close()
  await database.disconnect()
  console.info('>>>>>> 爬虫结束')
}

async function main() {
  if(outputPath){
    db.find({ "infoIndex": { $gte: startIndex , $lte: endIndex} }, function (err, docs) {
      if(err){
        console.error(err) 
      } else {
        var body = `infoIndex;who;phone;zone;workType;comment\n`
        for(var doc of docs){
          body += `${doc.infoIndex};${(doc.who || '').trim()};${doc.phone};${(doc.zone || '').trim()};${(doc.workType || '').trim()};${(doc.comment || '').trim()}\n`
        }
        fs.writeFileSync(options.export, body)
      }
      exit()
    });
  } else {
    await search()
  }
  console.info('>>>>>> 总找到:', totalFound)
}

main()

步骤3：完善database连接模块 #

确保./config/database.js有正确的连接和断开逻辑：

const mongoose = require('mongoose');

const connection = async () => {
  try {
    await mongoose.connect('你的MongoDB连接字符串', {
      useNewUrlParser: true,
      useUnifiedTopology: true
    });
    console.log('MongoDB连接成功');
  } catch (err) {
    console.error('MongoDB连接失败:', err);
    process.exit(1);
  }
};

const disconnect = async () => {
  try {
    await mongoose.connection.close();
    console.log('MongoDB连接已关闭');
  } catch (err) {
    console.error('关闭MongoDB连接失败:', err);
  }
};

module.exports = { connection, disconnect };

关键修正点说明 #

1. 模型字段对齐：将爬虫中的info字段改为comment，补充date字段，手机号改用String类型
2. 作用域修正：将MongoDB写入逻辑放到爬虫循环内部，直接访问当前循环的contact和i
3. 连接逻辑优化：爬虫启动时连接MongoDB，结束后再断开，避免提前关闭连接
4. 错误处理添加：捕获MongoDB写入错误，便于排查问题
5. 代码冗余合并：将两个分类的处理逻辑合并，减少重复代码
6. 函数逻辑优化：修正页面检查函数的拼写和内部逻辑，避免多个Promise竞争问题

内容的提问来源于stack exchange，提问作者sino