如何使用Lucene索引摘要中含常见ngram重复的文档

以下方案兼容Lucene 4.x及以上版本，核心逻辑可直接复用。

1. 依赖引入（Maven示例） #

<!-- Lucene核心依赖 -->
<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-core</artifactId>
    <version>8.11.2</version> <!-- 4.x版本可替换为对应版本号 -->
</dependency>
<dependency>
    <groupId>org.apache.lucene</groupId>
    <artifactId>lucene-analyzers-common</artifactId>
    <version>8.11.2</version>
</dependency>

2. 自定义摘要NGram分词器 #

根据你需要检测的重复片段长度配置ngram的最小/最大长度，示例默认检测2-4长度的ngram重复：

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.ngram.NGramTokenizer;

public class SummaryNGramAnalyzer extends Analyzer {
    private final int minGram;
    private final int maxGram;

    public SummaryNGramAnalyzer(int minGram, int maxGram) {
        this.minGram = minGram;
        this.maxGram = maxGram;
    }

    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
        NGramTokenizer tokenizer = new NGramTokenizer(minGram, maxGram);
        // 统一转小写，避免大小写差异导致的统计误差
        return new TokenStreamComponents(tokenizer, new LowerCaseFilter(tokenizer));
    }
}

3. 构建文章与摘要索引 #

索引时同时存储原文章信息、对摘要做ngram分词索引：

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.FSDirectory;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;

// 入参dataList为你的数据集，每个元素包含articleId、articleContent、summary三个字段
public void buildSummaryIndex(List<Map<String, String>> dataList, String indexSavePath) throws Exception {
    // 初始化分词器，按需调整ngram长度范围
    SummaryNGramAnalyzer analyzer = new SummaryNGramAnalyzer(2, 4);
    IndexWriterConfig writerConfig = new IndexWriterConfig(analyzer);
    // 本地磁盘存储索引，内存存储可替换为new RAMDirectory()
    FSDirectory indexDir = FSDirectory.open(Paths.get(indexSavePath));
    IndexWriter indexWriter = new IndexWriter(indexDir, writerConfig);

    for (Map<String, String> item : dataList) {
        Document doc = new Document();
        // 存储原文章信息，不会被分词，仅用于查询后返回
        doc.add(new StoredField("articleId", item.get("articleId")));
        doc.add(new StoredField("articleContent", item.get("articleContent")));
        // 摘要字段开启ngram分词索引，同时存储term向量用于后续频率统计
        doc.add(new TextField("summary", item.get("summary"), Field.Store.YES, Field.TermVector.YES));
        indexWriter.addDocument(doc);
    }
    indexWriter.commit();
    indexWriter.close();
    indexDir.close();
}

4. 查询所有含重复NGram的文章 #

先统计全索引中出现次数>1的ngram，再用这些ngram检索对应的所有文章：

import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

public List<Document> getDuplicateNGramArticles(String indexSavePath) throws Exception {
    FSDirectory indexDir = FSDirectory.open(Paths.get(indexSavePath));
    IndexReader indexReader = DirectoryReader.open(indexDir);
    IndexSearcher searcher = new IndexSearcher(indexReader);
    List<String> duplicateNGrams = new ArrayList<>();

    // 第一步：遍历所有摘要的ngram，筛选出现次数大于1的重复ngram
    for (int docIdx = 0; docIdx < indexReader.maxDoc(); docIdx++) {
        Terms terms = indexReader.getTermVector(docIdx, "summary");
        if (terms == null) continue;
        TermsEnum termsEnum = terms.iterator();
        BytesRef termBytes;
        while ((termBytes = termsEnum.next()) != null) {
            String ngram = termBytes.utf8ToString();
            int occurCount = indexReader.docFreq(new Term("summary", ngram));
            if (occurCount > 1 && !duplicateNGrams.contains(ngram)) {
                duplicateNGrams.add(ngram);
            }
        }
    }

    // 第二步：用所有重复ngram构造查询，返回匹配的所有文章
    BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
    for (String ngram : duplicateNGrams) {
        queryBuilder.add(new TermQuery(new Term("summary", ngram)), BooleanClause.Occur.SHOULD);
    }
    TopDocs topDocs = searcher.search(queryBuilder.build(), Integer.MAX_VALUE);
    List<Document> result = new ArrayList<>();
    for (int i = 0; i < topDocs.scoreDocs.length; i++) {
        result.add(searcher.doc(topDocs.scoreDocs[i].doc));
    }

    indexReader.close();
    indexDir.close();
    return result;
}

注意事项 #

• 若处理中文摘要，可在分词器中额外加入中文分词、标点过滤的Filter，避免无意义的标点符号ngram被统计
• 若需要检测更长的重复片段，直接修改SummaryNGramAnalyzer初始化时的minGram和maxGram参数即可
• Lucene 4.x版本仅部分API类名、参数存在差异，核心的分词、索引、查询逻辑无需修改，按需调整类引用即可正常运行

内容的提问来源于stack exchange，提问作者Kiera.K