net.java.sen
Class Tokenizer

java.lang.Object
  extended bynet.java.sen.Tokenizer
Direct Known Subclasses:
JapaneseTokenizer

public abstract class Tokenizer
extends java.lang.Object


Field Summary
 Node bosNode
           
 CToken bosToken
           
 CToken bosToken2
           
 Dictionary dic
           
 CToken eosToken
           
 CToken unknownToken
           
 
Constructor Summary
Tokenizer(java.lang.String tokenFile, java.lang.String doubleArrayFile, java.lang.String posInfoFile, java.lang.String connectFile, java.lang.String charset)
           
 
Method Summary
 void clear()
           
 boolean close()
           
abstract  int getCharClass(char c)
           
 Node getNewNode()
           
abstract  Node lookup(char[] c, int begin)
           
 int skipCharClass(char[] c, int begin, int end, int char_class)
           
 int skipCharClass(char[] c, int begin, int end, int char_class, int[] fail)
           
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

dic

public Dictionary dic

bosToken

public CToken bosToken

bosToken2

public CToken bosToken2

eosToken

public CToken eosToken

unknownToken

public CToken unknownToken

bosNode

public Node bosNode
Constructor Detail

Tokenizer

public Tokenizer(java.lang.String tokenFile,
                 java.lang.String doubleArrayFile,
                 java.lang.String posInfoFile,
                 java.lang.String connectFile,
                 java.lang.String charset)
          throws java.io.IOException
Method Detail

skipCharClass

public int skipCharClass(char[] c,
                         int begin,
                         int end,
                         int char_class,
                         int[] fail)

skipCharClass

public int skipCharClass(char[] c,
                         int begin,
                         int end,
                         int char_class)

getCharClass

public abstract int getCharClass(char c)

lookup

public abstract Node lookup(char[] c,
                            int begin)
                     throws java.io.IOException
Throws:
java.io.IOException

clear

public void clear()

getNewNode

public Node getNewNode()

close

public boolean close()