调用外部.py文件中cleaner函数时Jupyter内核崩溃求助

我正在开展基于WiFi CSI数据的手势检测研究，需要录制样本并截取包含手势的20数据包长度区间。编写了基于ipywidgets的脚本，用于显示原始频谱图、输入序列起始索引、展示裁剪后的频谱图，确认后将清洗数据写入文件。脚本从.py文件导入至.ipynb笔记本运行交互界面，但每次调用导入的cleaner函数时，Jupyter内核都会崩溃，该问题在本地机器和Google Colab上均出现。

相关代码 #

data_cleaner.py #

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import csiread # exports csv directly from pcap file!!
import ipywidgets as widgets # --> user interaction in Jupyter
from IPython.display import clear_output, display

def cleaner(csifile, gesture_length):
    # Extract csi directly from pcap file
    print('testing...') # not even this runs...
    csifile = '/content/REU/csi samples/csi_palmfist_1.pcap'
    csidata = csiread.Nexmon(csifile, chip='4339', bw=20)
    csidata.read()
    df1 = pd.DataFrame(csidata.csi)
    
    magMatrix = df1.applymap(abs) # magnitude matrix
    
    # define the useless subcarriers
    garbage = [1, 0, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38] # 14 removed
    
    cleaned = df1.drop(garbage, axis = 1)
    # display(cleaned)
    
    # amplitude visualization
    magMatrix = cleaned.applymap(np.abs)
    subcarLineChart(magMatrix)
    subcarSpectrogram(magMatrix)
    
    # smoothen the magnitude chart
    # apply is essentially map but passes each row instead of each element
    magSmoothened = magMatrix.apply(lambda x: mov_avg(x, 3))
    subcarLineChart(magSmoothened)
    subcarSpectrogram(magSmoothened)
    
    
    # UI STARTS HERE
    # limit useful gesture length to predefined value
    
    start_index = widgets.Text(
        value = '',
        description = 'Index for first packet in {}-packet sequence containing gesture:'.format(length)
        )
    extract_btn = widgets.Button(
        desciption = 'Extract'
        )
    
    extract_btn.on_click(handle_extract)
    
    # START = 
    
    # crop selected interval
    # btw: mag means magnitude
    # magSelected = magSmoothened[START:(START+LENGTH)]
    
    # show selected interval
    # subcarLineChart(magSelected)
    # subcarSpectrogram(magSelected)
    

## UI LOGIC
def handle_extract():
    magSelected = magSmoothened[START:(START+LENGTH)]
    subcarLineChart(magSelected)
    subcarSpectrogram(magSelected)
    
    display("Does this match what you were trying to extract? \n    If not, re-run the program. (ctrl-f9 on windows)\n")
    confirm_btn = widgets.Button(
        description = 'Yes, looks good'
        )
    confirm_btn.on_click(handle_confirm)
    
def handle_confirm():
    print('writes to file...')

## VISUALIZATION FUNCTIONS
from matplotlib import colors

def subcarLineChart(gph, log = False): # log: use log scale or not
    packet_index = len(gph.index)
    
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    plt.title("Magnitude of Signal by Channel and Packet Number")
    plt.xlabel("Packet #")
    plt.ylabel("Magnitude")
    ax1.set_yscale("log") if log else None
    
    num_subcarriers = len(gph.columns)
    num_samples = len(gph)
    for subcar_index in range(num_subcarriers):
        ax1.plot(
            range(num_samples),
            gph.iloc[:,subcar_index],
            lw = .25,
            label = subcar_index
        )
    
    plt.show()

def subcarSpectrogram(gph, log = False, subcar_start = 0, subcar_end = -1): # log: use log scale or not
    subcar_end = len(gph.iloc[0]) if subcar_end == -1 else subcar_end # replace -1 with valid final index
    plt.imshow(
    gph,
    # extent --> left, rigth, bottom, top
    extent = (subcar_start, subcar_end, len(gph), 0),
    norm = colors.LogNorm() if log else None)
    plt.xlabel('Subcarriers')
    plt.ylabel('Packet #')
    plt.colorbar(
      label = 'intensity')
    
    plt.show()
    
def mov_avg(arr, width):
    return np.convolve(arr, np.ones(width), 'valid') / width

interactive data cleaning.ipynb #

!pip install csiread

from google.colab import drive
import sys
drive.mount('/AshkanDrive')
!ln -s '/AshkanDrive/MyDrive/google_colab_files_for_CSI' '/content/REU'
filepath = '/content/REU/csi samples/old_testing_data/csi_palmfist_1.pcap'

sys.path.insert(0, '/content/REU/utils/')
from data_cleaner import cleaner

cleaner(filepath, 20) # 20 packets should be fine to fit all gestures

排查与修复方案 #

1. 修复变量作用域问题 #

• handle_extract、handle_confirm无法访问cleaner函数内的magSmoothened、gesture_length等变量，会引发未定义错误，严重时导致内核崩溃。
• 解决：将UI逻辑函数嵌套在cleaner内部，通过闭包直接访问内部变量，避免全局作用域混乱。

2. 释放绘图资源 #

• 连续调用plt.show()但未清理绘图资源，大量未关闭的Figure会占用内存，最终导致内核因内存耗尽崩溃。
• 解决：每次绘图后调用plt.close()释放资源；在Colab中先执行%matplotlib inline确保绘图正确渲染并自动释放资源。

3. 修正未定义变量与拼写错误 #

• cleaner函数中length未定义，应改为传入的gesture_length参数；START变量未定义，需从start_index输入框获取值。
• extract_btn的desciption拼写错误，应改为description，否则按钮无法正常渲染。

4. 提升数据处理效率 #

• df1.applymap(abs)和apply逐元素操作效率极低，数据量大时会占用大量CPU和内存。
• 解决：改用numpy向量化操作替代pandas的逐元素遍历，比如magMatrix = np.abs(cleaned.values)。

修复后的核心代码片段 #

def cleaner(csifile, gesture_length):
    print('testing...')
    csifile = '/content/REU/csi samples/csi_palmfist_1.pcap'
    csidata = csiread.Nexmon(csifile, chip='4339', bw=20)
    csidata.read()
    df1 = pd.DataFrame(csidata.csi)
    
    # 改用numpy向量化计算幅度
    garbage = [1, 0, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38]
    cleaned = df1.drop(garbage, axis=1)
    magMatrix_cleaned = np.abs(cleaned.values)
    magMatrix_cleaned_df = pd.DataFrame(magMatrix_cleaned)
    
    # 绘图后关闭资源
    subcarLineChart(magMatrix_cleaned_df)
    plt.close()
    subcarSpectrogram(magMatrix_cleaned_df)
    plt.close()
    
    # 向量化平滑处理
    def mov_avg(arr, width):
        return np.convolve(arr, np.ones(width), 'valid') / width
    magSmoothened = np.apply_along_axis(lambda x: mov_avg(x, 3), axis=0, arr=magMatrix_cleaned)
    magSmoothened_df = pd.DataFrame(magSmoothened)
    
    subcarLineChart(magSmoothened_df)
    plt.close()
    subcarSpectrogram(magSmoothened_df)
    plt.close()
    
    # UI部分
    start_index = widgets.Text(
        value='',
        description=f'Index for first packet in {gesture_length}-packet sequence containing gesture:'
    )
    
    # 嵌套定义处理函数，确保访问内部变量
    def handle_extract(b):
        try:
            START = int(start_index.value)
            magSelected = magSmoothened_df.iloc[START:START+gesture_length]
            subcarLineChart(magSelected)
            plt.close()
            subcarSpectrogram(magSelected)
            plt.close()
            
            display("Does this match what you were trying to extract? \n    If not, re-run the program. (ctrl-f9 on windows)\n")
            confirm_btn = widgets.Button(description='Yes, looks good')
            
            def handle_confirm(b):
                print('writes to file...')
                # 添加写入文件逻辑，例如 magSelected.to_csv('cleaned_data.csv')
            
            confirm_btn.on_click(handle_confirm)
            display(confirm_btn)
        except ValueError:
            display("请输入有效的整数索引！")
    
    extract_btn = widgets.Button(description='Extract')
    extract_btn.on_click(handle_extract)
    display(start_index, extract_btn)

额外建议 #

• 处理大型CSI数据时，优先用numpy操作替代pandas的逐元素遍历，减少内存占用。
• 添加异常处理（如文件不存在、索引超出范围等），避免意外错误导致内核崩溃。

内容的提问来源于stack exchange，提问作者Ashkan Arabi