如何在Express.js项目中实现服务器文件夹CSV实时自动导入mLab？

Got it, let's break this down step by step. You want to automatically import new CSV files dropped into a folder to mLab (a hosted MongoDB service) without command-line tools, and integrate this into an Express.js project. Here's a practical, production-ready approach:

First, we'll use these Node.js packages to make this work smoothly:

• chokidar: Reliable cross-platform file system watcher (way better than native fs.watch for edge cases like partial file writes)
• csv-parser: Lightweight streaming CSV parser (handles large files without eating up memory)
• mongodb: Official MongoDB driver (use mongoose instead if you prefer an ODM for schema validation)
• express: Our web framework to wrap this functionality
• dotenv: To keep sensitive credentials (like mLab connection strings) secure

Install them with:

npm install chokidar csv-parser mongodb express dotenv

Create a dedicated module (e.g., csvImporter.js) to handle the core logic. This will watch your target folder, parse new CSVs, and sync data to mLab.

const chokidar = require('chokidar');
const fs = require('fs');
const csv = require('csv-parser');
const { MongoClient } = require('mongodb');
require('dotenv').config();

// Configuration
const WATCH_FOLDER = './csv-uploads'; // Replace with your target folder path
const MLAB_URI = process.env.MLAB_URI; // Get this from your mLab dashboard
const DB_NAME = 'your-database-name';
const COLLECTION_NAME = 'your-target-collection';

// Track processed files to avoid re-importing duplicates
const processedFiles = new Set();

// Establish mLab connection once (reuse for all imports)
let dbClient;
async function connectToMLab() {
  try {
    dbClient = await MongoClient.connect(MLAB_URI, { 
      useNewUrlParser: true, 
      useUnifiedTopology: true 
    });
    console.log('Successfully connected to mLab');
  } catch (err) {
    console.error('Failed to connect to mLab:', err);
    process.exit(1); // Exit if we can't connect to the database
  }
}

// Parse CSV file and import rows to mLab
async function importCSV(filePath) {
  // Skip if file was already processed
  if (processedFiles.has(filePath)) return;
  processedFiles.add(filePath);

  const csvRows = [];
  return new Promise((resolve, reject) => {
    fs.createReadStream(filePath)
      .pipe(csv()) // Stream parse the CSV
      .on('data', (row) => csvRows.push(row)) // Collect each row as JSON
      .on('end', async () => {
        try {
          const db = dbClient.db(DB_NAME);
          const collection = db.collection(COLLECTION_NAME);
          
          // Insert all rows at once (use insertOne if you need per-row processing)
          const insertResult = await collection.insertMany(csvRows);
          console.log(`Imported ${insertResult.insertedCount} rows from ${filePath}`);
          resolve(insertResult);
        } catch (err) {
          console.error(`Failed to import ${filePath}:`, err);
          reject(err);
        }
      })
      .on('error', (err) => {
        console.error(`Failed to parse CSV file ${filePath}:`, err);
        reject(err);
      });
  });
}

// Start watching the target folder
async function startWatcher() {
  await connectToMLab();
  
  const watcher = chokidar.watch(WATCH_FOLDER, {
    ignored: /^\./, // Ignore hidden files
    persistent: true,
    awaitWriteFinish: { // Wait until file is fully written before processing
      stabilityThreshold: 2000, // Wait 2s after last change
      pollInterval: 100
    }
  });

  watcher
    .on('add', async (filePath) => {
      // Only process .csv files
      if (filePath.toLowerCase().endsWith('.csv')) {
        console.log(`New CSV detected: ${filePath}`);
        await importCSV(filePath);
      }
    })
    .on('error', (err) => console.error('File watcher error:', err));

  console.log(`Now watching folder for new CSVs: ${WATCH_FOLDER}`);
}

module.exports = { startWatcher, processedFiles };

Wrap the importer logic into your Express app. Create app.js:

const express = require('express');
const { startWatcher, processedFiles } = require('./csvImporter');
const app = express();
const PORT = process.env.PORT || 3000;

// Optional: Add a route to check import status
app.get('/import-status', (req, res) => {
  res.json({
    watchedFolder: './csv-uploads',
    totalProcessedFiles: processedFiles.size,
    serverStatus: 'Running'
  });
});

// Start the file watcher when the Express server boots
async function startServer() {
  await startWatcher();
  app.listen(PORT, () => {
    console.log(`Express server running on http://localhost:${PORT}`);
  });
}

startServer();

Create a .env file to store sensitive data (never hardcode these in your code!):

MLAB_URI=mongodb://your-username:your-password@dsxxxx.mlab.com:xxxx/your-database-name
PORT=3000

(Grab your mLab connection string from the mLab dashboard under "Connect" > "Connect using standard MongoDB URI")

• Duplicate Prevention: The processedFiles Set works for short runs, but for long-lived services, persist this list to a MongoDB collection to avoid losing state on server restart.
• Error Recovery: Add retries for failed imports, and move problematic files to a "failed-uploads" folder for later review.
• Permissions: Ensure the server process has read access to the watched folder and write access if you want to archive/delete files after import.
• Large Files: The streaming approach with csv-parser handles huge CSVs without loading the entire file into memory—critical for scalability.
• Security: Restrict mLab database access to your server's IP address (via mLab dashboard) and never commit .env to version control.

To run the app:

node app.js

Drop a CSV into the csv-uploads folder, and it will automatically import to your mLab collection!

内容的提问来源于stack exchange，提问作者humanity wins the race