Hey there! Let's work through this problem together. First off, I think you might have mixed up how populate() works in Mongoose—its job is to resolve references between collections (like pulling in a related document via a ref field), not to copy data from one collection to another. For your use case (syncing data from dht and co2 into excel every 20 minutes), we need a different approach using static methods + data insertion logic.

Here's a step-by-step solution:

1. Define Your Excel Schema with a Static Population Method #

First, set up your excel.model.js with a static method that fetches data from the other two collections and syncs it over. We'll handle both full syncs (clearing old data) or incremental syncs (only adding new data) depending on your needs.

const mongoose = require('mongoose');

// Define your Excel schema - adjust fields to match your dht/co2 data structure
const excelSchema = new mongoose.Schema({
  dhtRecord: { type: mongoose.Schema.Types.Mixed }, // Store full dht document or specific fields
  co2Record: { type: mongoose.Schema.Types.Mixed }, // Same for co2
  syncTimestamp: { type: Date, default: Date.now }
});

// Add the static method to populate the Excel collection
excelSchema.statics.syncFromDhtAndCo2 = async function(options = { fullSync: false }) {
  try {
    // Get references to the other models
    const DhtModel = mongoose.model('Dht');
    const Co2Model = mongoose.model('Co2');

    // Fetch data - use last sync time for incremental syncs
    let query = {};
    if (!options.fullSync) {
      const lastSync = await this.findOne().sort({ syncTimestamp: -1 });
      if (lastSync) {
        query = { createdAt: { $gt: lastSync.syncTimestamp } }; // Assume your dht/co2 docs have a createdAt field
      }
    }

    const newDhtDocs = await DhtModel.find(query);
    const newCo2Docs = await Co2Model.find(query);

    // Handle data matching (adjust this to your logic - e.g., pair by timestamp)
    const syncData = [];
    // Example: Pair dht and co2 records by their creation time (adjust if your matching logic is different)
    const pairedRecords = newDhtDocs.map(dhtDoc => {
      const matchingCo2 = newCo2Docs.find(co2Doc => 
        co2Doc.createdAt.getTime() - dhtDoc.createdAt.getTime() < 1000 // Match within 1 second
      );
      return {
        dhtRecord: dhtDoc,
        co2Record: matchingCo2 || null,
        syncTimestamp: new Date()
      };
    });

    // Add any unpaired co2 records too
    const unpairedCo2 = newCo2Docs.filter(co2Doc => 
      !newDhtDocs.some(dhtDoc => Math.abs(co2Doc.createdAt.getTime() - dhtDoc.createdAt.getTime()) < 1000)
    );
    unpairedCo2.forEach(co2Doc => {
      syncData.push({
        dhtRecord: null,
        co2Record: co2Doc,
        syncTimestamp: new Date()
      });
    });

    syncData.push(...pairedRecords);

    // Insert the data - clear old data if full sync is enabled
    if (options.fullSync) {
      await this.deleteMany({});
    }

    const result = await this.insertMany(syncData);
    console.log(`Synced ${result.length} records to Excel collection`);
    return result;
  } catch (error) {
    console.error('Failed to sync Excel collection:', error);
    throw error;
  }
};

const ExcelModel = mongoose.model('Excel', excelSchema);
module.exports = ExcelModel;

2. Set Up a Scheduler to Run Every 20 Minutes #

Since MQTT adds new data every 20 minutes, we'll use a cron job to trigger the sync automatically. You'll need the node-cron package for this (install it with npm install node-cron).

const ExcelModel = require('./excel.model');
const cron = require('node-cron');

// Schedule the sync to run every 20 minutes
cron.schedule('*/20 * * * *', async () => {
  console.log('Starting sync from DHT/CO2 to Excel collection...');
  try {
    // Use incremental sync by default - pass { fullSync: true } to refresh all data
    await ExcelModel.syncFromDhtAndCo2();
    console.log('Sync completed successfully!');
  } catch (err) {
    console.error('Sync failed:', err);
  }
});

Key Notes to Fix Your Original Approach #

• populate() isn't meant for copying data—it's for resolving references (e.g., if your Excel doc had a dhtId field pointing to a DHT record, populate() would fetch that record and replace the ID with the full document).
• For data sync, you need to explicitly fetch records from the source collections, process/match them as needed, then insert them into the target collection.
• Adjust the data matching logic (the part where we pair DHT and CO2 records) to fit your actual data structure—you might match by a shared timestamp, device ID, or another key.

内容的提问来源于stack exchange，提问作者Marks Gniteckis