如何通过Node.js实现Windows EC2与S3间文件/目录的可靠自动化传输？

Got it, your current setup is hitting SSH connection limits with just 5-6 concurrent users—makes total sense, since spinning up a new SSH connection for every request doesn’t scale at all. Let’s walk through some better Node.js-based solutions to handle file transfers between Windows EC2 instances and S3, with stability and concurrency in mind:

1. Fix SSH Scaling with Connection Pooling #

The core issue here is that each request creates a brand new SSH connection. Most SSH servers (including Windows OpenSSH on EC2) have default concurrent connection limits, and your ECS instance’s network resources will get tapped out quickly too.

Instead of spawning a new ssh2-sftp-client per request, use a connection pool to reuse existing connections. You can wrap the SFTP client with a generic pooling library like generic-pool:

First, install dependencies:

npm install ssh2-sftp-client generic-pool

Then implement the pool:

const { createPool } = require('generic-pool');
const Client = require('ssh2-sftp-client');

// Create a pool for each unique EC2 instance (reuse configs where possible)
function createSftpPool(hostConfig) {
  return createPool({
    create: async () => {
      const sftp = new Client();
      await sftp.connect({
        host: hostConfig.host,
        username: hostConfig.username,
        password: hostConfig.password,
        port: 22,
        tryKeyboard: true
      });
      return sftp;
    },
    destroy: async (sftp) => {
      try {
        await sftp.end();
      } catch (err) {
        console.error('Error closing SFTP connection:', err);
      }
    },
    validate: async (sftp) => {
      // Check if connection is still alive before reuse
      try {
        await sftp.list('.');
        return true;
      } catch (err) {
        return false;
      }
    }
  }, {
    max: 10, // Adjust based on your EC2's SSH connection limit
    min: 2, // Keep a few warm connections ready
    idleTimeoutMillis: 300000, // Close idle connections after 5 minutes
  });
}

// Manage pools for multiple EC2 instances
const ec2Pools = new Map();
function getPoolForEC2(host) {
  if (!ec2Pools.has(host)) {
    const config = getHostConfig(host); // Fetch your EC2 configs from a secure store
    ec2Pools.set(host, createSftpPool(config));
  }
  return ec2Pools.get(host);
}

// Updated file push handler using the pool
async function pushFileToEC2(host, file) {
  const pool = getPoolForEC2(host);
  let sftp;
  try {
    sftp = await pool.acquire();
    if (file.filename && file.file) {
      await sftp.put(file.file, `C:/Users/user/Desktop/${file.filename}`);
      console.log(`Successfully pushed ${file.filename} to ${host}`);
    }
  } catch (ex) {
    console.error(`SFTP error pushing to ${host}:`, ex);
    throw ex; // Let your request handler surface the error to the user
  } finally {
    if (sftp) {
      await pool.release(sftp); // Return connection to pool instead of closing it
    }
  }
}

This drastically cuts down on the number of open SSH connections, since you’re reusing existing ones instead of creating new ones for every request.

2. Ditch SSH Entirely with AWS Native Tools (More Stable & Secure) #

Since you’re working with AWS EC2 and S3, you can eliminate SSH entirely by using AWS Systems Manager (SSM) to run commands on your Windows EC2 instances. SSM handles all connection management, so you don’t have to worry about SSH limits, and you don’t need to expose port 22 to the internet.

First, make sure your Windows EC2 instances have the SSM Agent installed (pre-installed on most AWS Windows AMIs) and an IAM role with permissions for ssm:SendCommand and S3 access (to sync files).

Then use the AWS SDK for JavaScript v3 to trigger commands:

npm install @aws-sdk/client-ssm

Example: Sync S3 File to Windows EC2 #

const { SSMClient, SendCommandCommand } = require('@aws-sdk/client-ssm');

const ssmClient = new SSMClient({ region: 'your-aws-region' });

async function syncS3ToEC2(instanceId, s3FilePath, ec2DestinationPath) {
  // Use AWS CLI on Windows to copy from S3
  const command = `aws s3 cp ${s3FilePath} "${ec2DestinationPath}"`;

  const params = {
    InstanceIds: [instanceId],
    DocumentName: 'AWS-RunPowerShellScript',
    Parameters: {
      commands: [command]
    }
  };

  try {
    const response = await ssmClient.send(new SendCommandCommand(params));
    console.log(`Sync command sent to EC2 ${instanceId}:`, response.Command.CommandId);
    // Optional: Poll for command status if you need to confirm completion
  } catch (err) {
    console.error('Error sending SSM command:', err);
  }
}

Example: Sync EC2 Folder to S3 #

async function syncEC2FolderToS3(instanceId, ec2FolderPath, s3DestinationPath) {
  // Compress folder first (Windows-friendly) then sync to S3
  const tempZip = 'C:/Temp/ec2-folder.zip';
  const commands = [
    `Compress-Archive -Path "${ec2FolderPath}" -DestinationPath "${tempZip}" -Force`,
    `aws s3 cp "${tempZip}" "${s3DestinationPath}/ec2-folder.zip"`,
    `Remove-Item "${tempZip}" -Force`
  ];

  const params = {
    InstanceIds: [instanceId],
    DocumentName: 'AWS-RunPowerShellScript',
    Parameters: {
      commands: commands
    }
  };

  try {
    const response = await ssmClient.send(new SendCommandCommand(params));
    console.log(`Folder sync command sent to EC2 ${instanceId}:`, response.Command.CommandId);
  } catch (err) {
    console.error('Error syncing EC2 folder to S3:', err);
  }
}

This approach scales infinitely better because SSM handles the heavy lifting of connecting to EC2, and you avoid SSH connection limits entirely.

3. Add Concurrency Control (Even with Pooling) #

If you stick with SSH pooling, add concurrency limits to your request handlers to prevent overwhelming the pool or EC2 instances. Use p-limit to cap concurrent operations:

npm install p-limit

const pLimit = require('p-limit');
const limit = pLimit(5); // Allow 5 concurrent transfers per EC2 instance

// In your request handler
async function handleFilePush(req, res) {
  const { host, file } = req.body;
  try {
    await limit(() => pushFileToEC2(host, file));
    res.status(200).send('File pushed successfully');
  } catch (err) {
    res.status(500).send(`Error pushing file: ${err.message}`);
  }
}

4. Fix Your Error Handling #

Your original code has a few error handling gaps:

• The sftp.on('error') listener is added after connecting, so it misses early connection errors.
• You’re calling sftp.end() in both the error listener and finally, which can cause double-closing.

For standalone SFTP clients (if you don’t use pooling), fix it like this:

async function pushFileOldWay(host, file) {
  const sftp = new Client();
  try {
    // Add error listener BEFORE connecting
    sftp.on('error', (err) => {
      console.error('SFTP connection error:', err);
    });
    
    await sftp.connect({ /* your config */ });
    if (file.filename && file.file) {
      await sftp.put(file.file, `C:/Users/user/Desktop/${file.filename}`);
    }
  } catch (ex) {
    console.error('SFTP exception pushing files:', ex);
    throw ex;
  } finally {
    try {
      await sftp.end();
    } catch (err) {
      console.error('Error closing SFTP connection:', err);
    }
  }
}

内容的提问来源于stack exchange，提问作者SI2