Lucene4：获取中文分词结果，根据文本计算boost

package com.clzhang.sample.lucene;import java.io.*;import java.util.*;import org.apache.lucene.analysis.Analyzer;import com.chenlb.mmseg4j.Dictionary;import com.chenlb.mmseg4j.analysis.SimpleAnalyzer;import com.chenlb.mmseg4j.analysis.ComplexAnalyzer;import com.chenlb.mmseg4j.analysis.MaxWordAnalyzer;import org.wltea.analyzer.lucene.IKAnalyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;/** * 环境：Lucene 4.1版本/IKAnalyzer 2012 FF版本/mmseg4j 1.9版本 * 1.给定输入文本，获取中文拆分词结果； * 2.给定输入文本，对该文本按一定规则进行权重打分； * ??如：文本中包含指定关键词的频率越高，分值越高。 * @author Administrator * */public class AnalyzerTool { ???// mmseg4j字典路径 ???private static final String MMSEG4J_DICT_PATH = "C:\\solr\\news\\conf"; ???private static Dictionary dictionary = Dictionary.getInstance(MMSEG4J_DICT_PATH); ???????// 负面关键词信息，如果文本中包含这些词，那么该文本的打分值将变高。 ???private static List<String> lstNegativeWord; ???????static { ???????lstNegativeWord = new ArrayList<String>(); ???????????????// 下列词语必须存在于词典中：或者是分词器自带的词典，或者是自定义词典； ???????// 否则计算权重结果不准，因为有关键词没有被分词器拆分出来。 ???????lstNegativeWord.add("不雅"); ???????lstNegativeWord.add("被免"); ???????lstNegativeWord.add("偷拍"); ???} ???????/** ????* 测试各种解析器对同样文本的解析结果 ????* @param content ????* @throws Exception ????*/ ???public static void testAnalyzer(String content) throws Exception { ???????Analyzer analyzer = new IKAnalyzer(); // 等于new IKAnalyzer(false); ???????System.out.println("new IKAnalyzer()解析输出：" + getAnalyzedStr(analyzer, content)); ???????analyzer = new IKAnalyzer(true); ???????System.out.println("new IKAnalyzer(true)解析输出：" + getAnalyzedStr(analyzer, content)); ???????????????analyzer = new SimpleAnalyzer(dictionary); ???????System.out.println("new SimpleAnalyzer()解析输出：" + getAnalyzedStr(analyzer, content)); ???????analyzer = new ComplexAnalyzer(dictionary); ???????System.out.println("new ComplexAnalyzer()解析输出：" + getAnalyzedStr(analyzer, content)); ???????????????analyzer = new MaxWordAnalyzer(dictionary); ???????System.out.println("new MaxWordAnalyzer()解析输出：" + getAnalyzedStr(analyzer, content)); ???} ???????/** ????* 取得权重结果，规则：在输入字符串中查找关键词，关键词出现频率越多，权重越高 ????* @param str ????* @return ????* @throws Exception ????*/ ???public static float getBoost(String str) throws Exception { ???????float result = 1.0F; ???????????????// 默认解析器，可以更改为其它解析器 ???????Analyzer analyzer = new IKAnalyzer();// ???????Analyzer analyzer = new SimpleAnalyzer(dictionary); ???????List<String> list = getAnalyzedStr(analyzer, str); ???????for(String word: lstNegativeWord) { ???????????if(list.contains(word)) { ???????????????result += 10F; // 每出现一种负面关键词（不管出现几次），分值加10 ???????????} ?????????????????} ???????????????return result; ???} ???????/** ????* 调用分词器解析输入内容，将每个分词加入到List，然后返回此List ????* @param content ????* @return ????* @throws Exception ????*/ ???public static List<String> getAnalyzedStr(Analyzer analyzer, String content) throws Exception { ???????TokenStream stream = analyzer.tokenStream(null, new StringReader(content)); ???????CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); ???????????????List<String> result = new ArrayList<String>(); ???????while(stream.incrementToken()) { ???????????result.add(term.toString()); ???????} ???????????????return result; ???} ???public static void main(String[] args) throws Exception { ???????// 注意：亭湖新区/亭湖这两个词必须存在于IKAnalyzer/mmseg4j两个用户自定义词典中 ???????String content = "亭湖新区因不雅难过分视频被免官员国企老总名单公布"; ???????????????System.out.println("原文：" + content); ???????testAnalyzer(content); ???????System.out.println("默认解析器打分结果：" + getBoost(content)); ???}}