net.java.sen
Class Tokenizer
java.lang.Object
net.java.sen.Tokenizer
- Direct Known Subclasses:
- JapaneseTokenizer
- public abstract class Tokenizer
- extends java.lang.Object
Constructor Summary |
Tokenizer(java.lang.String tokenFile,
java.lang.String doubleArrayFile,
java.lang.String posInfoFile,
java.lang.String connectFile,
java.lang.String charset)
|
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
dic
public Dictionary dic
bosToken
public CToken bosToken
bosToken2
public CToken bosToken2
eosToken
public CToken eosToken
unknownToken
public CToken unknownToken
bosNode
public Node bosNode
Tokenizer
public Tokenizer(java.lang.String tokenFile,
java.lang.String doubleArrayFile,
java.lang.String posInfoFile,
java.lang.String connectFile,
java.lang.String charset)
throws java.io.IOException
skipCharClass
public int skipCharClass(char[] c,
int begin,
int end,
int char_class,
int[] fail)
skipCharClass
public int skipCharClass(char[] c,
int begin,
int end,
int char_class)
getCharClass
public abstract int getCharClass(char c)
lookup
public abstract Node lookup(char[] c,
int begin)
throws java.io.IOException
- Throws:
java.io.IOException
clear
public void clear()
getNewNode
public Node getNewNode()
close
public boolean close()