Python Flask应用读取Feather文件时的潜在内存泄漏问题排查
2026-7-23
问题
我开发了一个基于Flask的小型术语查询Web应用，部署在Heroku平台（使用Gunicorn提供服务）。该应用会将Feather文件（由脚本从CSV、XLSX文件转换生成，部分文件含10万+行）读取到pandas中，根据用户POST的输入术语进行查询。Feather文件总大小约70MB，但每次API调用读取文件后内存占用不会下降，后续调用持续增长，最终导致Heroku服务器崩溃。
已尝试的步骤：
- 按照相关建议使用
--preload参数并设置web concurrency为2 - 使用
gc.collect()回收内存（仅能释放少量内存，效果不明显） - 在API调用前预加载文件→直接触发Heroku内存配额超限错误
应用代码
from flask import Flask, request, jsonify, render_template import pandas as pd import gc #import memory_profiler as mp app = Flask(__name__) #@mp.profile def load_dataframes(target_lang): df_gl = pd.read_feather(f"data/feather/glossary_{target_lang}.ft") df_exc = pd.read_feather(f"data/feather/merged_exc_{target_lang}.ft") return df_gl, df_exc @app.route("/") def index(): return render_template("index.html") @app.route("/", methods=["POST"]) #@mp.profile def main(): term = request.json["term"] target_lang = request.json["target_lang"] exact_match_gl = request.json["exact_match_gl"] exact_match_exc = request.json["exact_match_exc"] try: print(f"Query: {term}. Target lang: {target_lang}") df_gl, df_exc = load_dataframes(target_lang) except Exception as e: print(e) raise if exact_match_gl == 1: print(f"Searching for '{term}' in glossary… Exact match: true") results_gl = df_gl[df_gl["term_en-US"] == term] else: print(f"Searching for '{term}' in glossary… Exact match: false") results_gl = df_gl[df_gl["term_en-US"].str.contains(term, case=False, na=False)] if exact_match_exc == 1: results_exc = df_exc[df_exc["Source Term"] == term] print(f"Searching for '{term}' in translations excerpts… Exact match: true") else: results_exc = df_exc[df_exc["Source Term"].str.contains(term, case=False, na=False)] print(f"Searching for '{term}' in translations excerpts… Exact match: false") if len(results_gl) > 0: gl_source = results_gl["term_en-US"].tolist() gl_translation = results_gl["term_" + target_lang].tolist() gl_source_pos = results_gl["pos_en-US"].str.split(n=1).str[1].str.lower().tolist() gl_target_pos = results_gl["pos_" + target_lang].str.split(n=1).str[1].str.lower().tolist() gl_source_def = [s[:1].lower() + s[1:] for s in results_gl["def_en-US"].str.split(n=1).str[1].tolist()] else: gl_source, gl_translation, gl_source_pos, gl_target_pos, gl_source_def = [''] * 5 if len(results_exc) > 0: exc_source = results_exc["Source Term"].tolist() exc_translation = results_exc["Translation"].tolist() exc_cat = results_exc["String Category"].fillna("unknown").tolist() exc_platform = results_exc["Platform"].fillna("unknown").tolist() exc_product = results_exc["Product"].fillna("unknown").tolist() exc_version = results_exc["Version"].fillna("unknown").tolist() else: exc_source, exc_translation, exc_cat, exc_platform, exc_product, exc_version = [''] * 6 del [[df_gl, df_exc]] gc.collect() df_gl = None df_exc = None # JSON Response response = {"gl_source": gl_source, "gl_translation": gl_translation, "gl_source_pos": gl_source_pos, "gl_target_pos": gl_target_pos, "gl_source_def": gl_source_def, "exc_source": exc_source, "exc_translation": exc_translation, "exc_cat": exc_cat, "exc_platform": exc_platform, "exc_product": exc_product, "exc_version": exc_version} return jsonify(response) if __name__ == "__main__": app.run()
内存分析日志片段
首次查询内存增长
Line # Mem usage Increment Occurrences Line Contents ============================================================= 17 81.0 MiB 81.0 MiB 1 @app.route("/", methods=["POST"]) 18 @mp.profile 19 def main(): 20 81.0 MiB 0.0 MiB 1 term = request.json["term"] 21 81.0 MiB 0.0 MiB 1 target_lang = request.json["target_lang"] 22 81.0 MiB 0.0 MiB 1 exact_match_gl = request.json["exact_match_gl"] 23 81.0 MiB 0.0 MiB 1 exact_match_exc = request.json["exact_match_exc"] 24 25 81.0 MiB 0.0 MiB 1 try: 26 81.0 MiB 0.0 MiB 1 print(f"Query: {term}. Target lang: {target_lang}") 27 362.2 MiB 281.2 MiB 1 df_gl, df_exc = load_dataframes(target_lang) 28 except Exception as e: 29 print(e) 30 raise
第二次查询内存持续增长
Query: test. Target lang: fr-FR Filename: C:\Users\spide\Documents\termic-main\search.py Line # Mem usage Increment Occurrences Line Contents ============================================================= 1 from flask import Flask, request, jsonify, render_template 2 import pandas as pd 3 import memory_profiler as mp 4 import gc 5 6 app = Flask(__name__) 7 366.9 MiB 366.9 MiB 1 8 @mp.profile 9 384.8 MiB 17.9 MiB 1 def load_dataframes(target_lang): 10 533.4 MiB 148.6 MiB 1 df_gl = pd.read_feather(f"data/feather/glossary_{target_lang}.ft") 11 533.4 MiB 0.0 MiB 1 df_exc = pd.read_feather(f"data/feather/merged_exc_{target_lang}.ft") 12 return df_gl, df_exc
gc.collect()效果有限
Line # Mem usage Increment Occurrences Line Contents ============================================================= 67 329.4 MiB -29.4 MiB 1 gc.collect()
偶发自动释放但整体居高不下
35 if len(results_gl) > 0: 36 1098.5 MiB 0.0 MiB 1 gl_source = results_gl["term_en-US"].tolist() 37 1098.7 MiB 0.2 MiB 1 gl_translation = results_gl["term_" + target_lang].tolist() 38 gl_source_pos = results_gl["pos_en-US"].str.split(n=1).str[1].str.lower().tolist() 39 1098.7 MiB 0.0 MiB 1 gl_target_pos = results_gl["pos_" + target_lang].str.split(n=1).str[1].str.lower().tolist() 40 gl_source_def = [s[:1].lower() + s[1:] for s in results_gl["def_en-US"].str.split(n=1).str[1].tolist()] 41 else: 42 gl_source, gl_translation, gl_source_pos, gl_target_pos, gl_source_def = [''] * 5 43 631.3 MiB -467.4 MiB 1 44 631.3 MiB 0.0 MiB 1 if len(results_exc) > 0: 45 exc_source = results_exc["Source Term"].tolist() 46 631.3 MiB 0.0 MiB 1 exc_translation = results_exc["Translation"].tolist() 47 631.3 MiB 0.0 MiB 1 exc_cat = results_exc["String Category"].fillna("unknown").tolist() 48 631.3 MiB 0.0 MiB 1 exc_platform = results_exc["Platform"].fillna("unknown").tolist() 49 631.4 MiB 0.1 MiB 1 exc_product = results_exc["Product"].fillna("unknown").tolist() 50 631.4 MiB 0.0 MiB 1 exc_version = results_exc["Version"].fillna("unknown").tolist() 51 456.4 MiB -175.0 MiB 221 else: 52 exc_source, exc_translation, exc_cat, exc_platform, exc_product, exc_version = [''] * 6 53 54 del [[df_gl, df_exc]] 55 456.4 MiB 0.0 MiB 1 gc.collect()
Heroku内存超限日志
2023-05-03T12:06:33.052510+00:00 heroku[web.1]: source=web.1 dyno=heroku.305321644.814f8ad6-3350-4d6d-934c-25f61f3013de sample#memory_total=92.37MB sample#memory_rss=77.64MB sample#memory_cache=14.73MB sample#memory_swap=0.00MB sample#memory_pgpgin=25972pages sample#memory_pgpgout=2325pages sample#memory_quota=512.00MB (…) 2023-05-03T12:21:44.199217+00:00 heroku[web.1]: source=web.1 dyno=heroku.305321644.814f8ad6-3350-4d6d-934c-25f61f3013de sample#memory_total=100.52MB sample#memory_rss=85.84MB sample#memory_cache=14.68MB sample#memory_swap=0.00MB sample#memory_pgpgin=28070pages sample#memory_pgpgout=2336pages sample#memory_quota=512.00MB 2023-05-03T12:21:55.402309+00:00 app[web.1]: Query: test. Target lang: fr-FR 2023-05-03T12:22:05.926772+00:00 heroku[web.1]: source=web.1 dyno=heroku.305321644.814f8ad6-3350-4d6d-934c-25f61f3013de sample#memory_total=426.72MB sample#memory_rss=413.42MB sample#memory_cache=0.01MB sample#memory_swap=13.30MB sample#memory_pgpgin=145870pages sample#memory_pgpgout=40033pages sample#memory_quota=512.00MB (…) 2023-05-03T12:23:02.203757+00:00 app[web.1]: Query: exactly. Target lang: fr-FR 2023-05-03T12:23:17.312190+00:00 heroku[web.1]: source=web.1 dyno=heroku.305321644.814f8ad6-3350-4d6d-934c-25f61f3013de sample#load_avg_1m=0.00 sample#load_avg_5m=0.00 sample#load_avg_15m=0.00 2023-05-03T12:23:17.361744+00:00 heroku[web.1]: source=web.1 dyno=heroku.305321644.814f8ad6-3350-4d6d-934c-25f61f3013de sample#memory_total=695.81MB sample#memory_rss=511.64MB sample#memory_cache=0.00MB sample#memory_swap=184.16MB sample#memory_pgpgin=207192pages sample#memory_pgpgout=90008pages sample#memory_quota=512.00MB 2023-05-03T12:23:17.383034+00:00 heroku[web.1]: Process running mem=695M(135.9%) 2023-05-03T12:23:17.388969+00:00 heroku[web.1]: Error R14 (Memory quota exceeded)
请问这是否确实是内存泄漏问题？还可采取哪些措施修复？
分析与解决方案
1. 是否是内存泄漏？
这大概率不是传统意义上的内存泄漏（对象被意外引用无法回收），而是pandas内存分配特性+Gunicorn多进程模型导致的内存累积：
- pandas读取Feather时会申请大块内存，DataFrame被删除后，Python垃圾回收标记了可释放内存，但操作系统可能不会立即回收给进程池，而是保留在进程的空闲内存池中，导致后续调用时内存占用看似持续增长。
- Gunicorn多进程模式下，每个worker都会重复加载数据，叠加内存池累积效应，容易触发Heroku内存配额限制。
2. 修复措施
优化内存回收逻辑
你的代码中del [[df_gl, df_exc]]是错误的，应直接删除变量本身，且变量赋值为None需在del之后：
# 替换原有的内存回收代码 del df_gl, df_exc df_gl = None df_exc = None gc.collect()
改用进程内缓存，避免重复加载
预加载内存超限是因为多worker重复加载数据导致内存翻倍，可在每个worker内缓存已加载的DataFrame：
# 全局维护缓存字典 data_cache = {} def load_dataframes(target_lang): if target_lang in data_cache: return data_cache[target_lang] # 只加载需要的列，减少内存占用 df_gl = pd.read_feather(f"data/feather/glossary_{target_lang}.ft", columns=["term_en-US", f"term_{target_lang}", "pos_en-US", f"pos_{target_lang}", "def_en-US"]) df_exc = pd.read_feather(f"data/feather/merged_exc_{target_lang}.ft", columns=["Source Term", "Translation", "String Category", "Platform", "Product", "Version"]) # 为查询列建立索引，加速查询 df_gl = df_gl.set_index("term_en-US", drop=False) df_exc = df_exc.set_index("Source Term", drop=False)
相关产品推荐
相关产品推荐
