Lucene系列四：Lucene提供的分词器、IKAnalyze中文分词器集成、扩展IKAnalyzer的停用词和新词

<dependency><groupId>org.apache.lucenegroupId><artifactId>lucene-coreartifactId><version>7.3.0version>dependency><dependency><groupId>org.apache.lucenegroupId><artifactId>lucene-analyzers-smartcnartifactId><version>7.3.0version>dependency>

package com.luceneanalyzer.use.standardanalyzer;import java.io.IOException;import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;/*** Lucene core模块中的 StandardAnalyzer英文分词器使用* 英文分词效果好&＃xff0c;中文分词效果不好* &＃64;author THINKPAD**/
public class LuceneStandardAnalyzerTest {private static void doToken(TokenStream ts) throws IOException {ts.reset();CharTermAttribute cta &＃61; ts.getAttribute(CharTermAttribute.class);while (ts.incrementToken()) {System.out.print(cta.toString() &＃43; "|");}System.out.println();ts.end();ts.close();}public static void main(String[] args) throws IOException {String etext &＃61; "Analysis is one of the main causes of slow indexing. Simply put, the more you analyze the slower analyze the indexing (in most cases).";String chineseText &＃61; "张三说的确实在理。";// Lucene core模块中的 StandardAnalyzer 英文分词器try (Analyzer ana &＃61; new StandardAnalyzer();) {TokenStream ts &＃61; ana.tokenStream("coent", etext);System.out.println("标准分词器&＃xff0c;英文分词效果&＃xff1a;");doToken(ts);ts &＃61; ana.tokenStream("content", chineseText);System.out.println("标准分词器&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);} catch (IOException e) {}}
}

标准分词器&＃xff0c;英文分词效果&＃xff1a;
analysis|one|main|causes|slow|indexing|simply|put|more|you|analyze|slower|analyze|indexing|most|cases|
标准分词器&＃xff0c;中文分词效果&＃xff1a;
张|三|说|的|确|实|在|理|

package com.luceneanalyzer.use.smartchineseanalyzer;import java.io.IOException;import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;/*** Lucene提供的中文分词器模块&＃xff0c;lucene-analyzers-smartcn:Lucene 的中文分词器 SmartChineseAnalyzer* 中英文分词效果都不好* * &＃64;author THINKPAD**/
public class LuceneSmartChineseAnalyzerTest {private static void doToken(TokenStream ts) throws IOException {ts.reset();CharTermAttribute cta &＃61; ts.getAttribute(CharTermAttribute.class);while (ts.incrementToken()) {System.out.print(cta.toString() &＃43; "|");}System.out.println();ts.end();ts.close();}public static void main(String[] args) throws IOException {String etext &＃61; "Analysis is one of the main causes of slow indexing. Simply put, the more you analyze the slower analyze the indexing (in most cases).";String chineseText &＃61; "张三说的确实在理。";// Lucene 的中文分词器 SmartChineseAnalyzertry (Analyzer smart &＃61; new SmartChineseAnalyzer()) {TokenStream ts &＃61; smart.tokenStream("content", etext);System.out.println("smart中文分词器&＃xff0c;英文分词效果&＃xff1a;");doToken(ts);ts &＃61; smart.tokenStream("content", chineseText);System.out.println("smart中文分词器&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);}}
}

smart中文分词器&＃xff0c;英文分词效果&＃xff1a;
analysi|is|on|of|the|main|caus|of|slow|index|simpli|put|the|more|you|analyz|the|slower|analyz|the|index|in|most|case|
smart中文分词器&＃xff0c;中文分词效果&＃xff1a;
张|三|说|的|确实|在|理|

&＃64;Overrideprotected TokenStreamComponents createComponents(String fieldName, final Reader in) {Tokenizer _IKTokenizer &＃61; new IKTokenizer(in, this.useSmart());return new TokenStreamComponents(_IKTokenizer);}

protected abstract TokenStreamComponents createComponents(String fieldName);

<dependency><groupId>org.apache.lucenegroupId><artifactId>lucene-coreartifactId><version>7.3.0version>dependency> <dependency><groupId>com.janeluogroupId><artifactId>ikanalyzerartifactId><version>2012_u6version><exclusions><exclusion><groupId>org.apache.lucenegroupId><artifactId>lucene-coreartifactId>exclusion><exclusion><groupId>org.apache.lucenegroupId><artifactId>lucene-queryparserartifactId>exclusion><exclusion><groupId>org.apache.lucenegroupId><artifactId>lucene-analyzers-commonartifactId>exclusion>exclusions>dependency>

package com.study.lucene.ikanalyzer.Integrated;import org.apache.lucene.analysis.Analyzer;/*** 因为Analyzer的createComponents方法API改变了需要重新实现分析器* &＃64;author THINKPAD**/
public class IKAnalyzer4Lucene7 extends Analyzer {private boolean useSmart &＃61; false;public IKAnalyzer4Lucene7() {this(false);}public IKAnalyzer4Lucene7(boolean useSmart) {super();this.useSmart &＃61; useSmart;}public boolean isUseSmart() {return useSmart;}public void setUseSmart(boolean useSmart) {this.useSmart &＃61; useSmart;}&＃64;Overrideprotected TokenStreamComponents createComponents(String fieldName) {IKTokenizer4Lucene7 tk &＃61; new IKTokenizer4Lucene7(this.useSmart);return new TokenStreamComponents(tk);}}

package com.study.lucene.ikanalyzer.Integrated;import java.io.IOException;import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;/*** 因为Analyzer的createComponents方法API改变了需要重新实现分词器* &＃64;author THINKPAD**/
public class IKTokenizer4Lucene7 extends Tokenizer {// IK分词器实现private IKSegmenter _IKImplement;// 词元文本属性private final CharTermAttribute termAtt;// 词元位移属性private final OffsetAttribute offsetAtt;// 词元分类属性&＃xff08;该属性分类参考org.wltea.analyzer.core.Lexeme中的分类常量&＃xff09;private final TypeAttribute typeAtt;// 记录最后一个词元的结束位置private int endPosition;/*** &＃64;param in* &＃64;param useSmart*/public IKTokenizer4Lucene7(boolean useSmart) {super();offsetAtt &＃61; addAttribute(OffsetAttribute.class);termAtt &＃61; addAttribute(CharTermAttribute.class);typeAtt &＃61; addAttribute(TypeAttribute.class);_IKImplement &＃61; new IKSegmenter(input, useSmart);}/** (non-Javadoc)* * &＃64;see org.apache.lucene.analysis.TokenStream#incrementToken()*/&＃64;Overridepublic boolean incrementToken() throws IOException {// 清除所有的词元属性
clearAttributes();Lexeme nextLexeme &＃61; _IKImplement.next();if (nextLexeme !&＃61; null) {// 将Lexeme转成Attributes// 设置词元文本
termAtt.append(nextLexeme.getLexemeText());// 设置词元长度
termAtt.setLength(nextLexeme.getLength());// 设置词元位移
offsetAtt.setOffset(nextLexeme.getBeginPosition(),nextLexeme.getEndPosition());// 记录分词的最后位置endPosition &＃61; nextLexeme.getEndPosition();// 记录词元分类
typeAtt.setType(nextLexeme.getLexemeTypeString());// 返会true告知还有下个词元return true;}// 返会false告知词元输出完毕return false;}/** (non-Javadoc)* * &＃64;see org.apache.lucene.analysis.Tokenizer#reset(java.io.Reader)*/&＃64;Overridepublic void reset() throws IOException {super.reset();_IKImplement.reset(input);}&＃64;Overridepublic final void end() {// set final offsetint finalOffset &＃61; correctOffset(this.endPosition);offsetAtt.setOffset(finalOffset, finalOffset);}
}

package com.study.lucene.ikanalyzer.Integrated;import java.io.IOException;import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;/*** IKAnalyzer分词器集成测试:* 细粒度切分&＃xff1a;把词分到最细* 智能切分&＃xff1a;根据词库进行拆分符合我们的语言习惯* * &＃64;author THINKPAD**/
public class IKAnalyzerTest {private static void doToken(TokenStream ts) throws IOException {ts.reset();CharTermAttribute cta &＃61; ts.getAttribute(CharTermAttribute.class);while (ts.incrementToken()) {System.out.print(cta.toString() &＃43; "|");}System.out.println();ts.end();ts.close();}public static void main(String[] args) throws IOException {String etext &＃61; "Analysis is one of the main causes of slow indexing. Simply put, the more you analyze the slower analyze the indexing (in most cases).";String chineseText &＃61; "张三说的确实在理。";/*** ikanalyzer 中文分词器 因为Analyzer的createComponents方法API改变了 需要我们自己实现* 分析器IKAnalyzer4Lucene7和分词器IKTokenizer4Lucene7*/// IKAnalyzer 细粒度切分try (Analyzer ik &＃61; new IKAnalyzer4Lucene7();) {TokenStream ts &＃61; ik.tokenStream("content", etext);System.out.println("IKAnalyzer中文分词器 细粒度切分&＃xff0c;英文分词效果&＃xff1a;");doToken(ts);ts &＃61; ik.tokenStream("content", chineseText);System.out.println("IKAnalyzer中文分词器 细粒度切分&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);}// IKAnalyzer 智能切分try (Analyzer ik &＃61; new IKAnalyzer4Lucene7(true);) {TokenStream ts &＃61; ik.tokenStream("content", etext);System.out.println("IKAnalyzer中文分词器 智能切分&＃xff0c;英文分词效果&＃xff1a;");doToken(ts);ts &＃61; ik.tokenStream("content", chineseText);System.out.println("IKAnalyzer中文分词器 智能切分&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);}}
}

IKAnalyzer中文分词器 细粒度切分&＃xff0c;英文分词效果&＃xff1a;
analysis|is|one|of|the|main|causes|of|slow|indexing.|indexing|simply|put|the|more|you|analyze|the|slower|analyze|the|indexing|in|most|cases|
IKAnalyzer中文分词器 细粒度切分&＃xff0c;中文分词效果&＃xff1a;
张三|三|说的|的确|的|确实|实在|在理|
IKAnalyzer中文分词器 智能切分&＃xff0c;英文分词效果&＃xff1a;
analysis|is|one|of|the|main|causes|of|slow|indexing.|simply|put|the|more|you|analyze|the|slower|analyze|the|indexing|in|most|cases|
IKAnalyzer中文分词器 智能切分&＃xff0c;中文分词效果&＃xff1a;
张三|说的|确实|在理|

xml version&＃61;"1.0" encoding&＃61;"UTF-8"?>
DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties> <comment>IK Analyzer 扩展配置comment><entry key&＃61;"ext_stopwords">my_ext_stopword.dicentry>
properties>

package com.study.lucene.ikanalyzer.Integrated.ext;import java.io.IOException;import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import com.study.lucene.ikanalyzer.Integrated.IKAnalyzer4Lucene7;/*** 扩展 IKAnalyzer的词典测试* **/
public class ExtendedIKAnalyzerDicTest {private static void doToken(TokenStream ts) throws IOException {ts.reset();CharTermAttribute cta &＃61; ts.getAttribute(CharTermAttribute.class);while (ts.incrementToken()) {System.out.print(cta.toString() &＃43; "|");}System.out.println();ts.end();ts.close();}public static void main(String[] args) throws IOException {String chineseText &＃61; "厉害了我的国一经播出&＃xff0c;受到各方好评&＃xff0c;强烈激发了国人的爱国之情、自豪感&＃xff01;";// IKAnalyzer 细粒度切分try (Analyzer ik &＃61; new IKAnalyzer4Lucene7();) {TokenStream ts &＃61; ik.tokenStream("content", chineseText);System.out.println("IKAnalyzer中文分词器 细粒度切分&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);}// IKAnalyzer 智能切分try (Analyzer ik &＃61; new IKAnalyzer4Lucene7(true);) {TokenStream ts &＃61; ik.tokenStream("content", chineseText);System.out.println("IKAnalyzer中文分词器 智能切分&＃xff0c;中文分词效果&＃xff1a;");doToken(ts);}}
}

xml version&＃61;"1.0" encoding&＃61;"UTF-8"?>
DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties> <comment>IK Analyzer 扩展配置comment><entry key&＃61;"ext_dict">ext.dicentry> <entry key&＃61;"ext_stopwords">my_ext_stopword.dicentry>
properties>