gate.creole.annic.apache.lucene.analysis.standard
Class StandardTokenizerTokenManager

java.lang.Object
  gate.creole.annic.apache.lucene.analysis.standard.StandardTokenizerTokenManager

All Implemented Interfaces:

StandardTokenizerConstants

public class StandardTokenizerTokenManager
extends Object
implements StandardTokenizerConstants

Field Summary

protected char	curChar
(package private) int	curLexState
PrintStream	debugStream
(package private) int	defaultLexState
protected CharStream	input_stream
(package private) static long[]	jjbitVec0
(package private) static long[]	jjbitVec10
(package private) static long[]	jjbitVec11
(package private) static long[]	jjbitVec12
(package private) static long[]	jjbitVec13
(package private) static long[]	jjbitVec14
(package private) static long[]	jjbitVec2
(package private) static long[]	jjbitVec3
(package private) static long[]	jjbitVec4
(package private) static long[]	jjbitVec5
(package private) static long[]	jjbitVec6
(package private) static long[]	jjbitVec7
(package private) static long[]	jjbitVec8
(package private) static long[]	jjbitVec9
(package private) int	jjmatchedKind
(package private) int	jjmatchedPos
(package private) int	jjnewStateCnt
(package private) static int[]	jjnextStates
(package private) int	jjround
private int[]	jjrounds
private int[]	jjstateSet
static String[]	jjstrLiteralImages
(package private) static long[]	jjtoSkip
(package private) static long[]	jjtoToken
static String[]	lexStateNames

Fields inherited from interface gate.creole.annic.apache.lucene.analysis.standard.StandardTokenizerConstants

ACRONYM, ALPHA, ALPHANUM, APOSTROPHE, CJK, COMPANY, DEFAULT, DIGIT, EMAIL, EOF, HAS_DIGIT, HOST, LETTER, NOISE, NUM, P, tokenImage

Constructor Summary

StandardTokenizerTokenManager(CharStream stream)

StandardTokenizerTokenManager(CharStream stream, int lexState)

Method Summary

Token	getNextToken()
private void	jjAddStates(int start, int end)
private static boolean	jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
private static boolean	jjCanMove_1(int hiByte, int i1, int i2, long l1, long l2)
private static boolean	jjCanMove_2(int hiByte, int i1, int i2, long l1, long l2)
private void	jjCheckNAdd(int state)
private void	jjCheckNAddStates(int start)
private void	jjCheckNAddStates(int start, int end)
private void	jjCheckNAddTwoStates(int state1, int state2)
protected Token	jjFillToken()
private int	jjMoveNfa_0(int startState, int curPos)
private int	jjMoveStringLiteralDfa0_0()
void	ReInit(CharStream stream)
void	ReInit(CharStream stream, int lexState)
private void	ReInitRounds()
void	setDebugStream(PrintStream ds)
void	SwitchTo(int lexState)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

debugStream

public PrintStream debugStream

jjbitVec0

static final long[] jjbitVec0

jjbitVec2

static final long[] jjbitVec2

jjbitVec3

static final long[] jjbitVec3

jjbitVec4

static final long[] jjbitVec4

jjbitVec5

static final long[] jjbitVec5

jjbitVec6

static final long[] jjbitVec6

jjbitVec7

static final long[] jjbitVec7

jjbitVec8

static final long[] jjbitVec8

jjbitVec9

static final long[] jjbitVec9

jjbitVec10

static final long[] jjbitVec10

jjbitVec11

static final long[] jjbitVec11

jjbitVec12

static final long[] jjbitVec12

jjbitVec13

static final long[] jjbitVec13

jjbitVec14

static final long[] jjbitVec14

jjnextStates

static final int[] jjnextStates

jjstrLiteralImages

public static final String[] jjstrLiteralImages

lexStateNames

public static final String[] lexStateNames

jjtoToken

static final long[] jjtoToken

jjtoSkip

static final long[] jjtoSkip

input_stream

protected CharStream input_stream

jjrounds

private final int[] jjrounds

jjstateSet

private final int[] jjstateSet

curChar

protected char curChar

curLexState

int curLexState

defaultLexState

int defaultLexState

jjnewStateCnt

int jjnewStateCnt

jjround

int jjround

jjmatchedPos

int jjmatchedPos

jjmatchedKind

int jjmatchedKind

Constructor Detail

StandardTokenizerTokenManager

public StandardTokenizerTokenManager(CharStream stream)

StandardTokenizerTokenManager

public StandardTokenizerTokenManager(CharStream stream,
                                     int lexState)

Method Detail

setDebugStream

public void setDebugStream(PrintStream ds)

jjMoveStringLiteralDfa0_0

private final int jjMoveStringLiteralDfa0_0()

jjCheckNAdd

private final void jjCheckNAdd(int state)

jjAddStates

private final void jjAddStates(int start,
                               int end)

jjCheckNAddTwoStates

private final void jjCheckNAddTwoStates(int state1,
                                        int state2)

jjCheckNAddStates

private final void jjCheckNAddStates(int start,
                                     int end)

jjCheckNAddStates

private final void jjCheckNAddStates(int start)

jjMoveNfa_0

private final int jjMoveNfa_0(int startState,
                              int curPos)

jjCanMove_0

private static final boolean jjCanMove_0(int hiByte,
                                         int i1,
                                         int i2,
                                         long l1,
                                         long l2)

jjCanMove_1

private static final boolean jjCanMove_1(int hiByte,
                                         int i1,
                                         int i2,
                                         long l1,
                                         long l2)

jjCanMove_2

private static final boolean jjCanMove_2(int hiByte,
                                         int i1,
                                         int i2,
                                         long l1,
                                         long l2)

ReInit

public void ReInit(CharStream stream)

ReInitRounds

private final void ReInitRounds()

ReInit

public void ReInit(CharStream stream,
                   int lexState)

SwitchTo

public void SwitchTo(int lexState)

jjFillToken

protected Token jjFillToken()

getNextToken

public Token getNextToken()