如何优化将字符串列表转为层级DataFrame的低效函数？

需求说明 #

我有一个包含数千条字符串的列表，需将其整理为层级DataFrame：每行对应单个术语的完整路径，每列为可变层级，所有行中列x的字符串需以整词形式完整包含于列x+1的字符串（仅为单词内子串时不匹配）。示例输入如下：

terms = ["term a",
         "term a term b",
         "word",
         "word a term b",
         "words",
         "contains words",
         "term a contains",
         "term b contains words",
         "term a term b contains words"]

期望输出为指定结构的层级DataFrame。

现状 #

我已实现organize_terms函数满足需求，但处理大量数据时速度极慢，核心瓶颈为带边界的re.search操作。后续我用nltk的ngrams实现了search函数替代，速度有所提升，但仍希望进一步优化性能。现寻求该函数的优化或重构建议。

现有代码 #

organize_terms函数 #

import pandas as pd
import re
import itertools

def organize_terms(terms):

    levels = {}
    path = {}
    terms_copy = terms.copy()

    # 确定每个术语所属的层级
    count = 0

    while len(terms) > 0:

        levels[f"level_{count}"] = []

        for term in terms:
            # 找出当前术语集合中，比当前术语短且整词包含于当前术语的所有术语
            temp = [term_2 for term_2 in terms if re.search(r"\b{}\b".format(term_2), term) != None and len(term_2.split()) < len(term.split())]
            # 无匹配项则当前术语为当前层级的根节点
            if temp == []:
                levels[f"level_{count}"].append(term)
        
        # 移除已分配层级的术语，继续处理剩余内容
        terms = [x for x in terms if x not in levels[f"level_{count}"]]
        
        count += 1

    """
    levels 最终结构示例：
    {'level_0': ['term a', 'word', 'words'],
     'level_1': ['term a term b', 'word a term b', 'contains words', 'term a contains'],
     'level_2': ['term b contains words'],
     'level_3': ['term a term b contains words']}
    """

    # 初始化路径字典，每个层级对应空列表
    for key in levels.keys():
        path[key] = []

    # 为每个原始术语构建所有合法路径
    for term in terms_copy:

        # 收集每个层级中整词包含于当前术语的所有术语
        matches = []
        for level in levels.keys():
            level_match = [term_2 for term_2 in levels[level] if re.search(r"\b{}\b".format(term_2), term) != None]
            if level_match != []:
                matches.append(level_match)

        # 生成所有层级匹配术语的笛卡尔积（所有可能的路径组合）
        combinations = list(itertools.product(*matches))

        """
        combinations 示例：
        对于"term a"：[('term a',)]
        对于"term a term b"：[('term a', 'term a term b')]
        对于"term a term b contains words"：
        [('term a', 'term a term b', 'term b contains words', 'term a term b contains words'),
         ('term a', 'contains words', 'term b contains words', 'term a term b contains words'),
         ('words', 'term a term b', 'term b contains words', 'term a term b contains words'),
         ('words', 'contains words', 'term b contains words', 'term a term b contains words')]
        """

        # 验证路径组合是否满足层级递进关系（前一层术语整词包含于后一层）
        for combination in combinations:
            confirmation = all([re.search(r"\b{}\b".format(combination[i-1]), combination[i]) != None for i in range(1, len(combination))])
            if confirmation:
                list_combination = list(combination)
                # 补全路径到最大层级数，不足位置用None填充
                while len(list_combination) < len(levels.keys()):
                    list_combination.append(None)
                # 将路径各层级术语添加到对应列表
                for idx, combination_term in enumerate(list_combination):
                    path[f"level_{idx}"].append(combination_term)
         
    """
    path 最终结构示例：
    {'level_0': ['term a', 'term a', 'word', 'word', 'words', 'words', 'term a', 'words', 'words'],
     'level_1': [None, 'term a term b', None, 'word a term b', None, 'contains words', 'term a contains', 'contains words', 'contains words'],
     'level_2': [None, None, None, None, None, None, None, 'term b contains words', 'term b contains words'],
     'level_3': [None, None, None, None, None, None, None, None, 'term a term b contains words']}
    """

    # 转换为DataFrame并返回
    return pd.DataFrame(path)

替代search函数 #

from nltk import ngrams

def search(term1, term2):
    # 将term2按词分割后，生成长度等于term1词数的ngram并拼接为字符串
    ngram_list = [' '.join(tuples) for tuples in list(ngrams(term2.split(), len(term1.split())))]
    # 判断term1是否为term2的连续整词子串
    return term1 in ngram_list

内容的提问来源于stack exchange，提问作者Eduardo Ramon Resser