HanLP 关键词提取。入门篇

dependencies {
    compile ‘org.springframework.boot:spring-boot:2.0.5.RELEASE‘
    compile ‘org.springframework.boot:spring-boot-starter-web:2.0.5.RELEASE‘
    providedRuntime ‘org.springframework.boot:spring-boot-starter-tomcat:2.0.5.RELEASE‘
    testCompile group: ‘junit‘, name: ‘junit‘, version: ‘4.11‘
    compile ‘com.hankcs:hanlp:portable-1.7.3‘
}

group ‘keyword‘
version ‘1.0-SNAPSHOT‘
apply plugin: ‘java‘
apply plugin: ‘war‘
war {
    archiveName ‘key-word.war‘
}

　　@Test
    public void test0(){
        String text = "中国是世界上的经济大国，社会主义强国！"; 
　　　　　List<Term> keyWords = HanLP.segment(text);
　　　　  System.out.println(keyWords.toString()); 
　　}

public class Phrase implements Comparable<Phrase>{
    private String word;           //候选关键词
    private boolean inDictionary;  //是否在HanLP词典中
    private String suffixWord;     //中心词。对单词，中心词就是word。如果是短语，则是短语中最后一词
    private String suffixWordPos;  //中心词的词性
    private String prefixWordPos;  //首词的词性
    private int freqOfDict = 1;    //在词典中该词的词频
    private boolean single;        //true表示单词，false表示短语
    private Location location;     //该候选关键词出现的位置
    private int offset;            //该候选词在正文中位置
    private boolean isCandidate;   //是否候选关键词

    *
    *
    *
   /**
　　*这会在比较score时用到
　　*/
    @Override
    public int compareTo(Phrase o) {
        return this.getWord().compareTo(o.getWord());
    }
}

/**全局逻辑

先找出关键词，再计算分数

一、
     * 从分词Term中解析 候选的关键词，
     * 因为要计算每个句子的分数值需要句子里的所有词累加计算score，所以这里都需要打标分数。
     * @param terms              分词列表
     * @param title              文章标题
     * @param firstParagraphEnd  对于正文处理，表示正文第一段结尾位置
     * @param lastParagraphBegin 对于正文处理，表示正文最后一段开始位置
     */

int index= 0;

while (index < terms.size()) {
    //当前词
    Term current = terms.get(index);
    //首先判断是否是命名实体 人名开头 包含 地名 动名词 其他专名，或者团体名开头

if (current.nature.startsWith("nr")
                    || current.nature.equals(Nature.ns)
                    || current.nature.equals(Nature.nz)
                    || current.nature.equals(Nature.vn)
                    || current.nature.startsWith("nt")) {

                Phrase phrase = new Phrase(current.word,
                        CoreDictionary.contains(current.word),
                        current.word,
                        current.nature.toString(),
                        current.nature.toString(),
                        CoreDictionary.get(current.word) == null ? 3 : CoreDictionary.get(current.word).totalFrequency,
                        true,
                        title.contains(current.word) ? Location.TITLE : whichLocation(terms.get(index).offset, firstParagraphEnd, lastParagraphBegin),
                        terms.get(index).offset, true);
                phrases.add(phrase);
                index++;
            } 
二、

//计算候选关键词的权重
TreeMap<Phrase, Double> scoreMap = new TreeMap<>();
for (Phrase phrase : phrases) {
    double score = scoreMap.containsKey(phrase) ? scoreMap.get(phrase) : 0.0;

    //排除单字符 的关键字
    int wordLength = phrase.getWord().toString().length();
    if (!(wordLength == 1)) {
        score += Math.log(weight(phrase, content, title) * freqMap.get(phrase.getWord()) / phrase.getFreqOfDict());
        scoreMap.put(phrase, score);
    }
    //weight方法：根据词性、词频、位置、词语长度等因素计算权重
    //这里依赖HanLp核心词典实现TF*IDF算法
}
三、

倒排

Comparator<Map.Entry<Phrase, Double>> valueComparator = new Comparator<Map.Entry<Phrase, Double>>() {
    @Override
    public int compare(Map.Entry<Phrase, Double> o1,
                       Map.Entry<Phrase, Double> o2) {
        return (o2.getValue().compareTo(o1.getValue()));
    }
};
List<Map.Entry<Phrase, Double>> list = new ArrayList<Map.Entry<Phrase, Double>>(scoreMap.entrySet());
Collections.sort(list, valueComparator);

此刻就可利用subList函数 取出tonN的关键词了

subList(0, Math.min(topN, 候选关键词数组size()));

四、
在三的基础上，对正文按标点符号 。？！；.!? 等进行分句。

List<Sentence> sentences = new ArrayList<>();
int i = 0;
int lastSentenceEnd = 0;
while (i <= content.length() - 1) {
    char c = content.charAt(i);
    if (SENTENCE_END_TAGS.indexOf(c) >= 0
            && i > lastSentenceEnd + 1
            && i > 0
            && i < content.length()) {
        Location location = Location.MIDDLE;
        if (i < firstParagraphEnd) {
            location = Location.FIRST;
        } else if (i > lastParagraphBegin) {
            location = Location.LAST;
        }
        int begin = lastSentenceEnd + 1;
        if (sentences.isEmpty()) begin = 0; // 对第一句，应该从0开始
        Sentence sentence = new Sentence(begin, i, location);
        sentences.add(sentence);
        lastSentenceEnd = i;
    }
    i++;
}

五，分句后，对每句含有的词，在三的基础上，进行分数累加。（有个小逻辑：句子长度对分数比例的影响 || 单个句子包含多个命名实体 人名等对分数比例的影响。|| 。。。）这些需要大量的场景测for (Sentence sentence : sentences) {

    String line = content.substring(sentence.getBegin(), sentence.getEnd() + 1);
    double sumScore = wordsWeight.get(wordsWeight.size() - 1).getValue();
    //注意：要用关键词中最小权重做为基数，避免句子权重计算结果为0
    int maxSize = Math.min(topN, wordsWeight.size());
    for (int i = 0; i < maxSize; i++) {
        Map.Entry<Phrase, Double> weight = wordsWeight.get(i);
        double weightDouble = weight.getValue();
        if (line != null && line.indexOf(weight.getKey().getWord()) >= 0){
            List<Term> terms = NLPTokenizer.segment(line);
            int index =0;int termCount = 0;
　　　　　　　sumScore += weightDouble;}
  　　　　}
　　　　sentence.setScore(sumScore);

然后同理，根据subList()取出想要的几个句子。再根据句子所在正文的位置，进行一个先后顺序的排列。

Comparator<Sentence> valueComparator = new Comparator<Sentence>() {

    @Override
    public int compare(Sentence o1, Sentence o2) {
        return o1.getBegin() - o2.getBegin();
    }
};
Collections.sort(prefixSentences, valueComparator);
StringBuilder sb = new StringBuilder();
for (Sentence sentence : prefixSentences) {
    sb.append(SentenceTool.toString(sentence, content));
}