AWS Lambda读取RDS MySQL大数据量触发堆内存溢出问题求助
2026-8-22
问题：从RDS MySQL读取大量数据时触发JavaScript堆内存溢出错误
我在使用Lambda函数从RDS MySQL读取大量数据行时，遇到以下错误：FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
Lambda函数代码
import * as AWS from 'aws-sdk'; const Knex = require('knex'); //AWS.config.update({ region: 'us-east-1' }); const host = // 已隐藏; const user = // 已隐藏; const password = // 已隐藏; const database = // 已隐藏'sys'; const connection = { ssl: { rejectUnauthorized: false }, host, user, password, database, }; // 创建连接 const knex = Knex({ client:'mysql', connection, }); exports.handler = async () => { try { const res = await knex('ALM_AGGR_LQ_201231_210111_0600_A048_V3_75_GAI').select(); const response = { statusCode: 200, //body: JSON.stringify(res), }; return response; } catch(err) { console.error(err); } };
CloudWatch调用日志
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | timestamp | message | |---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 1660654927927 | START RequestId: b2783c75-715b-461c-9e45-3b0cd4e69b8a Version: $LATEST | | 1660655028047 | <--- Last few GCs --- | 1660655028047 | [9:0x55d25c605d40] 99339 ms: Scavenge (reduce) 9213.5 (9372.5) -> 9213.3 (9372.8) MB, 27.8 / 0.0 ms (average mu = 0.402, current mu = 0.385) allocation failure | | 1660655028047 | [9:0x55d25c605d40] 100420 ms: Mark-sweep (reduce) 9213.9 (9372.8) -> 9213.8 (9373.8) MB, 1037.4 / 0.0 ms (+ 1353.9 ms in 299 steps since start of marking, biggest step 15.2 ms, walltime since start of marking 3339 ms) (average mu = 0.402, current mu = | | 1660655028047 | <--- JS stacktrace --- | 1660655028047 | FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory | | 1660655028050 | 1: 0x55d257982213 node::Abort() [/var/lang/bin/node] | | 1660655028050 | 2: 0x55d25785dd9e node::FatalError(char const*, char const*) [/var/lang/bin/node] | | 1660655028051 | 3: 0x55d257bbe502 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/var/lang/bin/node] | | 1660655028051 | 4: 0x55d257bbe7be v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/var/lang/bin/node] | | 1660655028052 | 5: 0x55d257dcda78 [/var/lang/bin/node] | | 1660655028053 | 6: 0x55d257de003c v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/var/lang/bin/node] | | 1660655028053 | 7: 0x55d257de386b v8::internal::Heap::AllocateRawWithRetryOrFailSlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [/var/lang/bin/node] | | 1660655028054 | 8: 0x55d257d956bd v8::internal::Factory::AllocateRaw(int, v8::internal::AllocationType, v8::internal::AllocationAlignment) [/var/lang/bin/node] | | 1660655028054 | 9: 0x55d257d8c98b v8::internal::FactoryBase<v8::internal::Factory>::AllocateRawArray(int, v8::internal::AllocationType) [/var/lang/bin/node] | | 1660655028055 | 10: 0x55d257d8caa3 v8::internal::FactoryBase<v8::internal::Factory>::NewFixedArrayWithFiller(v8::internal::Handle<v8::internal::Map>, int, v8::internal::Handle<v8::internal::Oddball>, v8::internal::AllocationType) [/var/lang/bin/node] | | 1660655028055 | 11: 0x55d258066d40 v8::internal::Handle<v8::internal::NameDictionary> v8::internal::HashTable<v8::internal::NameDictionary, v8::internal::NameDictionaryShape>::NewInternal<v8::internal::Isolate>(v8::internal::Isolate*, int, v8::internal::AllocationType) [/var/lang/bin/node] | | 1660655028056 | 12: 0x55d2580671e4 v8::internal::Handle<v8::internal::NameDictionary> v8::internal::HashTable<v8::internal::NameDictionary, v8::internal::NameDictionaryShape>::EnsureCapacity<v8::internal::Isolate>(v8::internal::Isolate*, v8::internal::Handle<v8::internal::NameDictionary>, int, v8::internal::AllocationType) [/var/lang/bin/node] | | 1660655028057 | 13: 0x55d258067259 v8::internal::Handle<v8::internal::NameDictionary> v8::internal::Dictionary<v8::internal::NameDictionary, v8::internal::NameDictionaryShape>::Add<v8::internal::Isolate>(v8::internal::Isolate*, v8::internal::Handle<v8::internal::NameDictionary>, v8::internal::Handle<v8::internal::Name>, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyDetails, v8::internal::InternalIndex*) [/var/lang/bin/node] | | 1660655028057 | 14: 0x55d258067799 v8::internal::BaseNameDictionary<v8::internal::NameDictionary, v8::internal::NameDictionaryShape>::Add(v8::internal::Isolate*, v8::internal::Handle<v8::internal::NameDictionary>, v8::internal::Handle<v8::internal::Name>, v8::internal::Handle<v8::internal::Object>, v8::internal::PropertyDetails, v8::internal::InternalIndex*) [/var/lang/bin/node] | | 1660655028058 | 15: 0x55d2581bea09 v8::internal::Runtime_AddDictionaryProperty(int, unsigned long*, v8::internal::Isolate*) [/var/lang/bin/node] | | 1660655028058 | 16: 0x55d2586405f9 [/var/lang/bin/node] | | 1660655037369 | END RequestId: b2783c75-715b-461c-9e45-3b0cd4e69b8a | | 1660655037369 | REPORT RequestId: b2783c75-715b-461c-9e45-3b0cd4e69b8a Duration: 109436.88 ms Billed Duration: 109437 ms Memory Size: 10240 MB Max Memory Used: 10240 MB Init Duration: 376.53 ms | | 1660655037369 | RequestId: b2783c75-715b-461c-9e45-3b0cd4e69b8a Error: Runtime exited with error: signal: aborted Runtime.ExitError | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Lambda函数规格
- 内存：10240 MB
- 超时时间：10分钟
- 临时存储：512 MB
解决方案
1. 使用流式查询（分批读取）
当前代码一次性把所有数据加载到内存，直接导致堆溢出。Knex支持流式处理，通过stream()方法逐行读取数据，避免一次性占用大量内存：
exports.handler = async () => { try { return new Promise((resolve, reject) => { const stream = knex('ALM_AGGR_LQ_201231_210111_0600_A048_V3_75_GAI') .select() .stream(); stream.on('data', (row) => { // 处理单条数据，比如写入S3、发送到消息队列等 console.log('处理数据行:', row); // 处理后及时释放变量引用，帮助GC回收内存 }); stream.on('end', () => { resolve({ statusCode: 200, body: '数据处理完成' }); }); stream.on('error', (err) => { console.error('流式处理错误:', err); reject({ statusCode: 500, body: JSON.stringify(err) }); }); }); } catch(err) { console.error(err); return { statusCode: 500, body: JSON.stringify(err) }; } };
2. 分页查询
如果不需要流式处理，可以用分页方式，每次查询固定数量的行，循环处理：
exports.handler = async () => { try { const pageSize = 1000; let offset = 0; let hasMore = true; while (hasMore) { const rows = await knex('ALM_AGGR_LQ_201231_210111_0600_A048_V3_75_GAI') .select() .limit(pageSize) .offset(offset); if (rows.length === 0) { hasMore = false; break; } // 处理当前页的数据 console.log(`处理第${Math.floor(offset/pageSize) +1}页，共${rows.length}条数据`); // 手动触发GC（可选，帮助快速释放内存） if (global.gc) global.gc(); offset += pageSize; } return { statusCode: 200, body: '分页处理完成' }; } catch(err) { console.error(err); return { statusCode: 500, body: JSON.stringify(err) }; } };
注意：如果数据量极大，
offset过大会导致SQL查询性能下降，此时建议改用基于主键的范围查询（比如where('id', '>', lastId)）来优化分页逻辑。
3. 调整Node.js堆内存上限（临时缓解）
Lambda分配的内存中，Node.js默认堆内存占比约75%。可以通过设置环境变量NODE_OPTIONS="--max-old-space-size=7680"（对应10GB内存的75%）来提升堆内存上限，但这只是临时方案，无法从根本解决大量数据加载的问题。
4. 数据预处理到中间存储
如果需要频繁读取该数据集，建议先将数据导出到S3、Elasticsearch等适合批量读取的存储服务，后续从这些服务获取数据，避免直接从RDS读取超大数据集。
内容的提问来源于stack exchange，提问作者Boris Grunwald
相关产品推荐
相关产品推荐
