基于467k词库的英文检测代码处理190词文本耗时20秒求优化

我用以下Python代码判断文本是否为英文，使用的words.txt词库包含467k条英文单词，但传入190个单词的文本时检测耗时约20秒，远超预期的1-2秒，求更快的优化方案。

原代码：

import re
from collections import Counter

class LanguageEvaluator:
    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.file_path = english_words_file
        self.min_non_english_count = min_non_english_count
        self.english_words = set()

    async def load_english_words(self):
        if not self.english_words:
            with open(self.file_path, 'r', encoding='utf-8') as file:
                self.english_words = {word.strip().lower() for word in file}
        return self.english_words

    async def preprocess_text(self, text):
        words = re.findall(r'\b\w+\b', text.lower())
        return [word for word in words if len(word) >= self.min_word_len and not word.startswith('@') and not re.match(r'^https?://', word)]

    async def count_non_english_words(self, words):
        english_words = await self.load_english_words()
        return sum(1 for word in words if not any(english_word.startswith(word) for english_word in english_words))

    async def is_english_custom(self, text):
        words_in_text = await self.preprocess_text(text)
        non_english_count = await self.count_non_english_words(words_in_text)
        print(f"Non-English words count: {non_english_count}")
        return non_english_count <= self.min_non_english_count

    async def count_duplicate_words(self, text):
        words = await self.preprocess_text(text)
        word_counts = Counter(words)
        duplicate_count = sum(
            count - 1 for count in word_counts.values() if count > 1)
        return duplicate_count

核心性能瓶颈 #

原代码最大的问题是count_non_english_words方法中，对每个文本单词都遍历467k条词库单词做前缀匹配，时间复杂度为O(N*M)（N是文本单词数，M是词库大小），直接导致了20秒的耗时。

优化方案 #

1. 构建前缀树（Trie）加速前缀匹配 #

前缀树可将前缀匹配的时间复杂度降至O(K)（K为单词长度），大幅减少匹配耗时：

class TrieNode:
    def __init__(self):
        self.children = {}
        self.is_end = False

class Trie:
    def __init__(self):
        self.root = TrieNode()

    def insert(self, word):
        node = self.root
        for char in word:
            if char not in node.children:
                node.children[char] = TrieNode()
            node = node.children[char]
        node.is_end = True

    def has_prefix(self, prefix):
        node = self.root
        for char in prefix:
            if char not in node.children:
                return False
            node = node.children[char]
        return True

2. 同步加载词库并初始化前缀树 #

原代码的async异步加载无必要，反而增加调度开销，改为实例化时同步加载并构建前缀树。

3. 预编译正则表达式 #

预编译所有用到的正则表达式，避免每次调用方法时重复编译。

4. 提前终止检测 #

当统计的非英文单词数超过阈值min_non_english_count时，直接停止遍历并返回结果。

优化后的完整代码 #

import re
from collections import Counter

class TrieNode:
    def __init__(self):
        self.children = {}
        self.is_end = False

class Trie:
    def __init__(self):
        self.root = TrieNode()

    def insert(self, word):
        node = self.root
        for char in word:
            if char not in node.children:
                node.children[char] = TrieNode()
            node = node.children[char]
        node.is_end = True

    def has_prefix(self, prefix):
        node = self.root
        for char in prefix:
            if char not in node.children:
                return False
            node = node.children[char]
        return True

class LanguageEvaluator:
    # 预编译正则表达式
    WORD_PATTERN = re.compile(r'\b\w+\b')
    URL_PATTERN = re.compile(r'^https?://')

    def __init__(self, english_words_file='words.txt', min_word_len=4, min_non_english_count=4):
        self.min_word_len = min_word_len
        self.min_non_english_count = min_non_english_count
        self.trie = Trie()
        # 实例化时同步加载词库并构建前缀树
        self._load_english_words(english_words_file)

    def _load_english_words(self, file_path):
        with open(file_path, 'r', encoding='utf-8') as file:
            for line in file:
                word = line.strip().lower()
                self.trie.insert(word)

    def preprocess_text(self, text):
        words = self.WORD_PATTERN.findall(text.lower())
        filtered_words = []
        for word in words:
            if len(word) >= self.min_word_len and not word.startswith('@') and not self.URL_PATTERN.match(word):
                filtered_words.append(word)
        return filtered_words

    def count_non_english_words(self, words):
        non_english_count = 0
        for word in words:
            if not self.trie.has_prefix(word):
                non_english_count += 1
                # 提前终止：超过阈值就停止统计
                if non_english_count > self.min_non_english_count:
                    break
        return non_english_count

    def is_english_custom(self, text):
        words_in_text = self.preprocess_text(text)
        non_english_count = self.count_non_english_words(words_in_text)
        print(f"Non-English words count: {non_english_count}")
        return non_english_count <= self.min_non_english_count

    def count_duplicate_words(self, text):
        words = self.preprocess_text(text)
        word_counts = Counter(words)
        duplicate_count = sum(count - 1 for count in word_counts.values() if count > 1)
        return duplicate_count

额外优化建议 #

• 过滤词库中长度小于min_word_len的单词，减少前缀树的节点数量。
• 若只需要精确匹配而非前缀匹配，直接用set的O(1)查找性能会更优；前缀树仅在必须支持前缀匹配时使用。

内容的提问来源于stack exchange，提问作者Indrajeeth