Node.js中Puppeteer爬虫无法识别已连接客户端的问题

我正在使用Node.js、Express、Puppeteer和MongoDB开发Web应用，目标是实现一个通过Server-Sent Events（SSE）向已连接客户端推送实时更新的爬虫。但遇到问题：尽管客户端已连接服务器，爬虫脚本却无法正确识别这些客户端。

项目结构 #

• server.js：管理Express服务器、客户端连接与SSE
• notifyClients.js：通过Map对象管理客户端连接
• scraper.js：运行Puppeteer进行数据爬取并通知客户端

代码文件 #

notifyClients.js

const clientResponses = new Map();

const loadClients = () => {
    const clients = Array.from(clientResponses.keys());
    console.log(`Loaded clients: ${JSON.stringify(clients)}`);
    return clients;
};

const notifyClients = (message) => {
    const clients = loadClients();
    console.log('Notifying clients');
    clients.forEach((clientId) => {
        const client = clientResponses.get(clientId);
        if (client) {
            try {
                console.log(`Sending message to client ${clientId}: ${JSON.stringify(message)}`);
                client.write(`data: ${JSON.stringify(message)}

`);
            } catch (error) {
                console.error(`Failed to send message to client ${clientId}:`, error);
            }
        }
    });
};

const addClient = (res) => {
    const clientId = Date.now().toString(); // Generate a unique ID for the client
    clientResponses.set(clientId, res);
    console.log(`Client connected with ID ${clientId}, total clients: ${clientResponses.size}`);

    res.on('close', () => {
        clientResponses.delete(clientId);
        console.log(`Client with ID ${clientId} disconnected, total clients: ${clientResponses.size}`);
    });
};

const getClientCount = () => {
    const count = clientResponses.size;
    console.log(`Getting client count: ${count}`);
    return count;
};

module.exports = {
    notifyClients,
    addClient,
    getClientCount,
    clientResponses  // Export clientResponses for use in server.js
};

server.js

const express = require('express');
const mongoose = require('mongoose');
const cors = require('cors');
const bcrypt = require('bcryptjs');
const jwt = require('jsonwebtoken');
const { v4: uuidv4 } = require('uuid');
require('dotenv').config();
const path = require('path');
const { addClient, notifyClients, getClientCount, clientResponses } = require('./src/hooks/notifyClients');

const app = express();
const port = process.env.PORT || 5000;
const SECRET_KEY = process.env.SECRET_KEY || 'your_secret_key';

app.use(cors({
  origin: 'http://localhost:3000', // Frontend URL
  credentials: true,
}));
app.use(express.json());

mongoose.connect(process.env.MONGODB_URI, {
  useNewUrlParser: true,
  useUnifiedTopology: true,
})
  .then(() => console.log('MongoDB Connected'))
  .catch((error) => console.error('MongoDB connection error:', error));

// Other schema definitions and middlewares

// SSE endpoint to send updates to the client
app.get('/api/products/updates', (req, res) => {
  console.log('SSE endpoint hit');
  res.setHeader('Content-Type', 'text/event-stream');
  res.setHeader('Cache-Control', 'no-cache');
  res.setHeader('Connection', 'keep-alive');
  res.flushHeaders();

  addClient(res); // Use addClient to manage the client connections
});

app.get('/api/check-clients', (req, res) => {
  const clients = Array.from(clientResponses.keys());
  res.json({
    clientCount: clients.length,
    clients: clients.map((clientId, index) => `Client ${index + 1}: ${clientId}`)
  });
});

// Other routes and server startup code

app.listen(port, () => {
  console.log(`Server is running on port ${port}`);
});

scraper.js

const puppeteer = require('puppeteer');
const mongoose = require('mongoose');
const dotenv = require('dotenv');
const fs = require('fs');
const path = require('path');
const { notifyClients, getClientCount, clients } = require('../hooks/notifyClients');
const axios = require('axios');

dotenv.config();

// MongoDB connection and other configurations

// Add a helper function to wait until clients are connected
const waitForClients = async (timeout = 30000, interval = 1000) => {
  const startTime = Date.now();
  while (Date.now() - startTime < timeout) {
    const clientCount = getClientCount();
    console.log(`Waiting for clients. Current client count: ${clientCount}`);
    console.log(`Current clients array: ${JSON.stringify(clients)}`); // Log the current clients array for debugging

    // Perform the HTTP request to check client count
    try {
      const response = await axios.get('http://localhost:5000/api/check-clients');
      console.log(`HTTP request response: ${JSON.stringify(response.data)}`);
    } catch (error) {
      console.error(`HTTP request error: ${error.message}`);
    }

    if (clientCount > 0) {
      return true;
    }
    await new Promise(resolve => setTimeout(resolve, interval));
  }
  console.log('No clients connected within the timeout period.');
  return false;
};

(async () => {
  const browser = await puppeteer.launch();

  try {
    // Other scraping logic

    console.log('Waiting for clients to connect...');
    const clientsConnected = await waitForClients();
    if (!clientsConnected) {
      console.log('No clients connected within the timeout period.');
      return;
    }

    // Proceed with scraping and notifying clients
  } catch (error) {
    console.error('An error occurred:', error);
  } finally {
    await browser.close();
    mongoose.connection.close();
  }
})();

日志信息 #

爬虫日志 #

Waiting for clients. Current client count: 0
Current clients array: undefined
HTTP request response: {"clientCount":2,"clients":["Client 1: 1719764974304","Client 2: 1719764991201"]}
Getting client count: 0

服务器日志 #

Server is running on port 5000
SSE endpoint hit
Client connected with ID 1719764974304, total clients: 1
MongoDB Connected

客户端日志 #

Setting up EventSource connection
EventSource connection established

已尝试的操作 #

• 验证客户端已正确添加到notifyClients.js的clientResponses中
• 在server.js和scraper.js中添加日志调试客户端连接状态
• 确保clientResponses映射在server.js和scraper.js中正确导入使用

核心原因 #

scraper.js是独立的Node.js进程，和运行server.js的进程不共享内存。当你在爬虫进程中导入notifyClients.js时，会创建一个全新的clientResponses Map实例，和服务器进程中维护的那个完全无关，因此爬虫无法识别已连接的客户端。

修复方案 #

方案1：将爬虫整合到服务器进程中 #

不要单独启动scraper.js，而是把爬虫逻辑作为server.js的一部分，通过定时任务或API触发：

// server.js 新增爬虫触发逻辑
const puppeteer = require('puppeteer');

const runScraper = async () => {
  const browser = await puppeteer.launch();
  try {
    // 你的爬虫逻辑...
    const scrapedData = {}; // 替换为实际爬取的数据
    notifyClients(scrapedData);
  } catch (err) {
    console.error('Scraper error:', err);
  } finally {
    await browser.close();
  }
};

// 通过API触发爬虫
app.post('/api/scraper/run', async (req, res) => {
  await runScraper();
  res.json({ success: true });
});

方案2：让爬虫通过HTTP请求调用服务器的通知接口 #

如果必须单独运行爬虫进程，让爬虫通过HTTP请求通知服务器，由服务器负责推送给客户端：

1. 在server.js中新增接收爬虫通知的接口：

// server.js
app.post('/api/scraper/send-update', express.json(), (req, res) => {
  const updateData = req.body;
  notifyClients(updateData);
  res.status(200).json({ status: 'success', message: 'Update pushed to clients' });
});

2. 修改scraper.js，移除对notifyClients的导入，改用HTTP请求发送通知：

// scraper.js
// 移除原有导入：const { notifyClients, getClientCount, clients } = require('../hooks/notifyClients');

// 新增发送通知的函数
const sendUpdateToServer = async (data) => {
  try {
    await axios.post('http://localhost:5000/api/scraper/send-update', data);
    console.log('Update sent to server for client push');
  } catch (error) {
    console.error('Failed to send update to server:', error);
  }
};

// 调整等待客户端的逻辑，只依赖HTTP请求结果
const waitForClients = async (timeout = 30000, interval = 1000) => {
  const startTime = Date.now();
  while (Date.now() - startTime < timeout) {
    try {
      const response = await axios.get('http://localhost:5000/api/check-clients');
      const clientCount = response.data.clientCount;
      console.log(`Waiting for clients. Current client count: ${clientCount}`);
      
      if (clientCount > 0) {
        return true;
      }
    } catch (error) {
      console.error(`HTTP request error: ${error.message}`);
    }
    await new Promise(resolve => setTimeout(resolve, interval));
  }
  console.log('No clients connected within the timeout period.');
  return false;
};

方案3：用共享存储管理客户端连接（进阶） #

如果需要更复杂的跨进程通信，可以用Redis等共享存储维护客户端连接状态，但前两个方案更适合当前场景。

额外修复点 #

删除scraper.js中访问未定义clients变量的日志代码，因为notifyClients.js并未导出该变量。

内容的提问来源于stack exchange，提问作者Botond Till