基于Node.js的邮箱有效性验证（检测收件箱存在）技术咨询

问题背景 #

我基于Node.js开发了邮箱有效性验证工具，逻辑包含格式校验、MX记录解析、SMTP握手验证邮箱存在性，同时检测域名是否启用Catch-All。本地运行结果准确，但在远程Linux服务器上频繁出现连接错误，且需要处理约400万条邮箱列表，需解决连接问题并避免被目标服务器封禁。

现有实现代码 #

const { randomBytes } = require('crypto');
const promises = require('dns').promises;
const net = require('net');

const resolveMXRecords = async (domain) => {
    console.log('resolving mx records for domain...', domain)
    try {
        return await promises.resolveMx(domain)
    } catch (error) {
        return []
    }
}

const verifyEmailFormat = (email) => {
    const regex = new RegExp({ pattern: '^[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,4}$', flags: 'i' });
    return regex.test(email);
};

const SMTPStages = {
    CHECK_CONNECTION_ESTABLISHED: 'CHECK_CONNECTION_ESTABLISHED',
    SEND_EHLO: 'SEND_EHLO',
    SEND_MAIL_FROM: 'SEND_MAIL_FROM',
    SEND_RECIPIENT_TO: 'SEND_RECIPIENT_TO',
    SEND_QUIT: 'SEND_QUIT'
}

const testInboxOnSMTPServer = async (inboxAddress, smtpAddress) => {
    return new Promise((resolve, reject) => {
        const result = {
            connection_established: false,
            account_exists: false
        }

        const socket = net.createConnection(25, smtpAddress)
        const stages = {
            [SMTPStages.CHECK_CONNECTION_ESTABLISHED]: {
                expected_reply_code: '220'
            },
            [SMTPStages.SEND_EHLO]: {
                command: `EHLO mail.example.org\r\n`,
                expected_reply_code: '250'
            },
            [SMTPStages.SEND_MAIL_FROM]: {
                command: `MAIL FROM:<name@example.org>\r\n`,
                expected_reply_code: '250'
            },
            [SMTPStages.SEND_RECIPIENT_TO]: {
                command: `RCPT TO:<${inboxAddress}>\r\n`,
                expected_reply_code: '250'
            },
            [SMTPStages.SEND_QUIT]: {
                command: `QUIT\r\n`,
                expected_reply_code: '221'
            }
        }

        let response = ""
        let currentStageName = SMTPStages.CHECK_CONNECTION_ESTABLISHED

        socket.on("data", (data) => {
            const reply = data.toString()
            response += reply

            // console.log('<-- ' + reply)
            const currentStage = stages[currentStageName]

            switch (currentStageName) {
                case SMTPStages.CHECK_CONNECTION_ESTABLISHED: {
                    if (!reply.startsWith(currentStage.expected_reply_code)) {
                        socket.end()
                        break
                    }

                    result.connection_established = true
                    currentStageName = SMTPStages.SEND_EHLO
                    const command = stages[currentStageName].command
                    socket.write(command, () => {
                        console.log('--> ' + command)
                    })

                    break
                }

                case SMTPStages.SEND_EHLO: {
                    if (!reply.startsWith(currentStage.expected_reply_code)) {
                        socket.end()
                        break
                    }

                    currentStageName = SMTPStages.SEND_MAIL_FROM
                    const command = stages[currentStageName].command
                    socket.write(command, () => {
                        console.log('--> ' + command)
                    })

                    break
                }

                case SMTPStages.SEND_MAIL_FROM: {
                    if (!reply.startsWith(currentStage.expected_reply_code)) {
                        socket.end()
                        break
                    }

                    currentStageName = SMTPStages.SEND_RECIPIENT_TO
                    const command = stages[currentStageName].command
                    socket.write(command, () => {
                        console.log('--> ' + command)
                    })

                    break
                }

                case SMTPStages.SEND_RECIPIENT_TO: {
                    if (!reply.startsWith(currentStage.expected_reply_code)) {
                        socket.end()
                        break
                    }

                    result.account_exists = true
                    currentStageName = SMTPStages.SEND_QUIT
                    const command = stages[currentStageName].command
                    socket.write(command, () => {
                        console.log('--> ' + command)
                    })

                    break
                }
            }
        })

        socket.on("connect", () => {
            console.log("Connected to SMTP " + smtpAddress)
            console.log('Local address: ' + socket.localAddress + ' Local port: ' + socket.localPort);
            console.log('Remote address: ' + socket.remoteAddress + ' Remote port: ' + socket.remotePort);
        })

        socket.on("error", (error) => {
            // clearInterval(timeoutTimer)
            reject(error)
        })

        socket.on("close", () => {
            // clearInterval(timeoutTimer)
            resolve(result)
        })
    })
}

const validateEmail = async (email) => {
    const emailFormatValid = verifyEmailFormat(email)
    if (!emailFormatValid) {
        return 'invalid email format'
    }

    const [, domain] = email.split('@')
    const mxRecords = await resolveMXRecords(domain)
    const sortedMxRecords = mxRecords.sort((a, b) => a.priority - b.priority)
    console.log('sortedMxRecords', sortedMxRecords)

    let smtpResult = { connection_established: false, account_exists: false }
    let hostIndex = 0

    while (hostIndex < sortedMxRecords.length) {
        try {
            console.log('testing inbox on smtp server', sortedMxRecords[hostIndex].exchange)
            smtpResult = await testInboxOnSMTPServer(email, sortedMxRecords[hostIndex].exchange)
            if (!smtpResult.connection_established) {
                console.log('connection not established')
                hostIndex++
            }
            else {
                break
            }
        } catch (error) {
            console.error('error while testing inbox on smtp server', error)
            hostIndex++
        }
    }
    let usesCatchAll = false
    try {
        const testCatchEmail = `${randomBytes(20).toString('hex')}@${domain}`
        const testCatchAll = await testInboxOnSMTPServer(testCatchEmail, sortedMxRecords[hostIndex].exchange)
        usesCatchAll = testCatchAll.account_exists
    } catch (error) {
        console.error(error)
    }

    return {
        email_format_is_valid: emailFormatValid,
        usesCatchAll: usesCatchAll,
        ...smtpResult
    }

}

async function main() {
    let email = 'abc@liqueous.com'
    const result = await validateEmail(email)
    console.log('result', result)
}
main()

远程连接错误修复方案 #

1. 端口与协议适配 #

大部分云服务器默认封禁25端口，改用587端口（STARTTLS）或465端口（SSL/TLS）：

• 465端口需使用tls模块替代net：

const tls = require('tls');
const socket = tls.connect(465, smtpAddress, { rejectUnauthorized: false });

2. 添加超时机制 #

当前代码无超时处理，连接挂起会导致资源泄漏，添加15秒超时：

// 在testInboxOnSMTPServer函数创建socket后添加
socket.setTimeout(15000);
socket.on('timeout', () => {
  socket.destroy(new Error('SMTP connection timed out'));
});

3. DNS解析优化 #

远程服务器DNS可能存在解析失败问题，指定公共DNS并增加重试：

const dnsOptions = { servers: ['8.8.8.8', '8.8.4.4'] };
const resolveMXRecords = async (domain) => {
    try {
        return await promises.resolveMx(domain, dnsOptions)
    } catch (error) {
        // 重试一次
        try {
            await new Promise(resolve => setTimeout(resolve, 1000));
            return await promises.resolveMx(domain, dnsOptions)
        } catch (retryError) {
            return []
        }
    }
}

4. 服务器安全组配置 #

检查远程Linux服务器的出站规则，确保允许访问25/587/465端口；关闭本地防火墙或添加对应端口的出站规则。

避免被目标服务器封禁的策略 #

1. 严格控制请求速率与并发 #

• 使用p-limit库限制单IP并发数（建议≤5）：

const pLimit = require('p-limit');
const limit = pLimit(5);
// 批量处理时用limit包裹validateEmail
const tasks = emails.map(email => limit(() => validateEmail(email)));
await Promise.all(tasks);

• 针对同一域名的SMTP服务器，添加1-3秒随机延迟：

// 在testInboxOnSMTPServer调用前添加
await new Promise(resolve => setTimeout(resolve, Math.random() * 2000 + 1000));

2. 随机化请求标识 #

• 随机生成EHLO域名：

const randomEHLO = `${randomBytes(8).toString('hex')}.example.com`;
// 修改SEND_EHLO的command
command: `EHLO ${randomEHLO}\r\n`,

• 随机生成MAIL FROM地址：

const randomFrom = `${randomBytes(10).toString('hex')}@${randomEHLO}`;
command: `MAIL FROM:<${randomFrom}>\r\n`,

3. IP地址轮换 #

大规模检测必须使用代理IP池，示例使用socks代理：

const SocksProxyAgent = require('socks-proxy-agent');
const agent = new SocksProxyAgent('socks://user:pass@proxy-host:port');
const socket = tls.connect(465, smtpAddress, { agent, rejectUnauthorized: false });

4. 缓存与跳过策略 #

• 同一域名只检测一次Catch-All，缓存结果：

const catchAllCache = new Map();
// 修改validateEmail中的Catch-All检测部分
let usesCatchAll = false;
if (catchAllCache.has(domain)) {
  usesCatchAll = catchAllCache.get(domain);
} else {
  try {
    const testCatchEmail = `${randomBytes(20).toString('hex')}@${domain}`;
    const testCatchAll = await testInboxOnSMTPServer(testCatchEmail, sortedMxRecords[hostIndex].exchange);
    usesCatchAll = testCatchAll.account_exists;
    catchAllCache.set(domain, usesCatchAll);
  } catch (error) {
    console.error(error);
  }
}

• 记录返回5xx错误的域名，后续跳过该域名的所有邮箱检测。

大规模数据处理优化 #

1. 断点续传与持久化 #

将检测结果写入SQLite数据库，支持断点续传：

const sqlite3 = require('sqlite3').verbose();
const db = new sqlite3.Database('email_validation.db');
// 初始化表
db.run('CREATE TABLE IF NOT EXISTS results (email TEXT PRIMARY KEY, valid INTEGER, format_valid INTEGER, catch_all INTEGER)');
// 保存结果
db.run('INSERT OR REPLACE INTO results (email, valid, format_valid, catch_all) VALUES (?, ?, ?, ?)',
  [email, smtpResult.account_exists ? 1 : 0, emailFormatValid ? 1 : 0, usesCatchAll ? 1 : 0]);

2. 错误重试机制 #

对连接错误、超时等可恢复错误，添加3次以内的指数退避重试：

const retry = async (fn, retries = 3, delay = 1000) => {
  try {
    return await fn();
  } catch (error) {
    if (retries > 0) {
      await new Promise(resolve => setTimeout(resolve, delay));
      return retry(fn, retries - 1, delay * 2);
    }
    throw error;
  }
};
// 使用retry包裹testInboxOnSMTPServer调用
smtpResult = await retry(() => testInboxOnSMTPServer(email, sortedMxRecords[hostIndex].exchange));

3. 日志与监控 #

记录详细日志到文件，便于排查问题：

const fs = require('fs');
const logger = fs.createWriteStream('validation.log', { flags: 'a' });
// 记录日志
logger.write(`${new Date().toISOString()} - ${email} - ${JSON.stringify(result)}\n`);

内容的提问来源于stack exchange，提问作者saimarshadd