基于Stanford Core NLP/Deeplearning4J的Java NLP文本洞察POC技术问询

Stanford Core NLP #

优质官方文档 #

• 入门配置指南：覆盖环境搭建、核心NLP任务（实体识别、语义分析、文本摘要）的基础配置与调用逻辑
• 模块专项文档：针对每个功能模块（如指代消解、情感分析）提供详细参数说明与场景示例

简易代码示例 #

1. 文本摘要与语义实体提取

import edu.stanford.nlp.pipeline.*;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.summarization.Summarizer;
import edu.stanford.nlp.util.CoreMap;

import java.util.Properties;

public class StanfordNLPDemo {
    public static void main(String[] args) {
        // 初始化NLP管道，加载所需标注器
        Properties props = new Properties();
        props.setProperty("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref, summarize");
        
        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
        // 替换为从CSV/NoSQL读取的自定义文档内容
        String targetText = "你的自定义文档内容，例如产品说明书、用户反馈片段等";
        
        Annotation document = new Annotation(targetText);
        pipeline.annotate(document);
        
        // 生成指定句数的文本摘要
        Summarizer summarizer = new Summarizer(props);
        String summary = summarizer.summarize(targetText, 3);
        System.out.println("文本摘要：\n" + summary);
        
        // 提取文本中的命名实体（语义洞察核心内容）
        for (CoreMap sentence : document.get(CoreAnnotations.SentencesAnnotation.class)) {
            System.out.println("\n句子实体识别结果：");
            sentence.get(CoreAnnotations.TokensAnnotation.class).forEach(token -> {
                String word = token.get(CoreAnnotations.TextAnnotation.class);
                String nerTag = token.get(CoreAnnotations.NamedEntityTagAnnotation.class);
                if (!nerTag.equals("O")) { // 过滤非实体标签
                    System.out.println(word + " -> " + nerTag);
                }
            });
        }
    }
}

2. 用户查询语义匹配

import edu.stanford.nlp.semgraph.SemanticGraph;
import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations;
import edu.stanford.nlp.util.CoreMap;

public class QueryMatcher {
    private static final StanfordCoreNLP pipeline;

    static {
        Properties props = new Properties();
        props.setProperty("annotators", "tokenize, ssplit, parse");
        pipeline = new StanfordCoreNLP(props);
    }

    public static void matchQuery(String userQuery, String documentText) {
        Annotation queryAnnotation = new Annotation(userQuery);
        pipeline.annotate(queryAnnotation);
        Annotation docAnnotation = new Annotation(documentText);
        pipeline.annotate(docAnnotation);

        // 获取查询语句的语义依存根节点
        CoreMap querySentence = queryAnnotation.get(CoreAnnotations.SentencesAnnotation.class).get(0);
        SemanticGraph queryGraph = querySentence.get(SemanticGraphCoreAnnotations.EnhancedDependenciesAnnotation.class);
        String queryRoot = queryGraph.getFirstRoot().word();

        // 遍历文档句子，匹配核心谓语
        for (CoreMap sentence : docAnnotation.get(CoreAnnotations.SentencesAnnotation.class)) {
            SemanticGraph docGraph = sentence.get(SemanticGraphCoreAnnotations.EnhancedDependenciesAnnotation.class);
            String docRoot = docGraph.getFirstRoot().word();
            if (docRoot.equalsIgnoreCase(queryRoot)) {
                System.out.println("匹配到相关内容：\n" + sentence);
            }
        }
    }
}

Deeplearning4J #

优质官方文档 #

• NLP模块入门指南：讲解文本向量化、预训练模型加载、自定义模型训练流程，包含CSV/NoSQL数据源的读取方案
• 预训练模型手册：覆盖Word2Vec、BERT等模型在DL4J中的调用方法，适用于语义相似度计算、文本表征提取等场景

简易代码示例 #

1. 从CSV读取文档并提取语义向量

import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer;
import org.deeplearning4j.models.word2vec.Word2Vec;
import org.deeplearning4j.text.sentenceiterator.BasicLineIterator;
import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;
import org.nd4j.linalg.api.ndarray.INDArray;

import java.io.File;

public class DL4JSemanticExtraction {
    public static void main(String[] args) throws Exception {
        // 加载预训练Word2Vec模型（可自行训练或使用官方开源模型）
        Word2Vec word2Vec = WordVectorSerializer.readWord2VecModel(new File("path/to/word2vec/model"));
        
        // 从CSV读取文档（每行对应一个文档片段）
        BasicLineIterator lineIterator = new BasicLineIterator(new File("path/to/your/documents.csv"));
        DefaultTokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();
        tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor());

        // 生成文档语义向量（词向量平均值）
        while (lineIterator.hasNext()) {
            String docText = lineIterator.nextSentence();
            INDArray wordVectors = word2Vec.getWordVectorMatrix(tokenizerFactory.create(docText).getTokens());
            INDArray docVector = wordVectors.mean(0); // 平均词向量作为文档语义表征
            System.out.println("文档语义向量：\n" + docVector);
        }
    }
}

2. 用户查询语义相似度匹配

import org.nd4j.linalg.ops.transforms.Transforms;

public class DL4JQueryMatcher {
    private static final Word2Vec word2Vec;
    private static final DefaultTokenizerFactory tokenizerFactory;

    static {
        try {
            word2Vec = WordVectorSerializer.readWord2VecModel(new File("path/to/word2vec/model"));
            tokenizerFactory = new DefaultTokenizerFactory();
            tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor());
        } catch (Exception e) {
            throw new RuntimeException("模型加载失败", e);
        }
    }

    public static void matchSemanticQuery(String userQuery, String documentText) {
        // 生成查询与文档的语义向量
        INDArray queryVec = word2Vec.getWordVectorMatrix(tokenizerFactory.create(userQuery).getTokens()).mean(0);
        INDArray docVec = word2Vec.getWordVectorMatrix(tokenizerFactory.create(documentText).getTokens()).mean(0);

        // 计算余弦相似度
        double similarity = Transforms.cosineSim(queryVec, docVec);
        if (similarity > 0.7) { // 自定义相似度阈值
            System.out.println("查询与文档匹配度：" + similarity);
            System.out.println("相关文档内容：\n" + documentText);
        }
    }
}

内容的提问来源于stack exchange，提问作者Sabari