de.grogra.grammar

Class Tokenizer

java.lang.Object

de.grogra.grammar.Tokenizer

All Implemented Interfaces:

antlr.TokenStream

Direct Known Subclasses:

DTDTokenizer, LSYTokenizer

public class Tokenizer
extends java.lang.Object
implements antlr.TokenStream

Field Summary

Fields

Modifier and Type	Field and Description
static int	CREATE_TOKEN_LOCATOR
protected boolean	createTokenLocator
static int	ENABLE_TYPE_SUFFIX
protected boolean	enableTypeSuffix
static int	EOL_IS_SIGNIFICANT
protected boolean	eolIsSignificant
static int	EVALUATE_NUMBERS
protected boolean	evaluateNumbers
static int	FLOAT_IS_DEFAULT
protected boolean	floatIsDefault
static I18NBundle	I18N
static int	MINUS_IS_SIGN
protected boolean	minusIsSign
protected boolean	unicode
static int	UNICODE_ESCAPES

Constructor Summary

Constructors

Constructor and Description
Tokenizer(int flags)

Method Summary

Methods

Modifier and Type	Method and Description
void	addToken(int tokenId, java.lang.String string)
void	addToken(int tokenId, java.lang.String string, boolean identifierPrecedes)
void	addToken(Token token)
void	addToken(Token token, boolean identifierPrecedes)
void	addToken(Token token, java.lang.String string)
void	addToken(Token token, java.lang.String string, boolean identifierPrecedes)
void	consume(int type)
protected Token	convert(Token t)
protected void	copyTokenTablesFrom(Tokenizer source)
float	getFloat()
Input	getInput()
int	getInt()
NumberLiteral	getNumberToken(int allowedLiterals)
Token	getToken()
protected boolean	isIdentifierPart(char c)
protected boolean	isIdentifierStart(char c)
protected boolean	isWhitespace(char c)
antlr.Token	nextToken()
void	reset()
void	setInput(Input input)
void	setSource(java.io.Reader source)
void	setSource(java.io.Reader source, java.lang.String name)
void	ungetToken()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CREATE_TOKEN_LOCATOR

public static final int CREATE_TOKEN_LOCATOR

See Also:

Constant Field Values

createTokenLocator

protected final boolean createTokenLocator

ENABLE_TYPE_SUFFIX

public static final int ENABLE_TYPE_SUFFIX

See Also:

Constant Field Values

enableTypeSuffix

protected final boolean enableTypeSuffix

EOL_IS_SIGNIFICANT

public static final int EOL_IS_SIGNIFICANT

See Also:

Constant Field Values

eolIsSignificant

protected final boolean eolIsSignificant

EVALUATE_NUMBERS

public static final int EVALUATE_NUMBERS

See Also:

Constant Field Values

evaluateNumbers

protected final boolean evaluateNumbers

FLOAT_IS_DEFAULT

public static final int FLOAT_IS_DEFAULT

See Also:

Constant Field Values

floatIsDefault

protected final boolean floatIsDefault

I18N

public static final I18NBundle I18N

MINUS_IS_SIGN

public static final int MINUS_IS_SIGN

See Also:

Constant Field Values

minusIsSign

protected final boolean minusIsSign

unicode

protected final boolean unicode

UNICODE_ESCAPES

public static final int UNICODE_ESCAPES

See Also:

Constant Field Values

Constructor Detail

Tokenizer

public Tokenizer(int flags)

Method Detail

addToken

public final void addToken(int tokenId,
            java.lang.String string)

addToken

public final void addToken(int tokenId,
            java.lang.String string,
            boolean identifierPrecedes)

addToken

public final void addToken(Token token)

addToken

public final void addToken(Token token,
            boolean identifierPrecedes)

addToken

public final void addToken(Token token,
            java.lang.String string)

addToken

public final void addToken(Token token,
            java.lang.String string,
            boolean identifierPrecedes)

consume

public final void consume(int type)
                   throws java.io.IOException,
                          LexicalException,
                          UnexpectedTokenException

Throws:

java.io.IOException

LexicalException

UnexpectedTokenException

convert

protected Token convert(Token t)

copyTokenTablesFrom

protected void copyTokenTablesFrom(Tokenizer source)

getFloat

public final float getFloat()
                     throws java.io.IOException,
                            LexicalException

Throws:

java.io.IOException

LexicalException

getInput

public final Input getInput()

getInt

public final int getInt()
                 throws java.io.IOException,
                        LexicalException

Throws:

java.io.IOException

LexicalException

getNumberToken

public final NumberLiteral getNumberToken(int allowedLiterals)
                                   throws java.io.IOException,
                                          LexicalException

Throws:

java.io.IOException

LexicalException

getToken

public final Token getToken()
                     throws java.io.IOException,
                            LexicalException

Throws:

java.io.IOException

LexicalException

isIdentifierPart

protected boolean isIdentifierPart(char c)

isIdentifierStart

protected boolean isIdentifierStart(char c)

isWhitespace

protected boolean isWhitespace(char c)

nextToken

public antlr.Token nextToken()
                      throws antlr.TokenStreamException

Specified by:

nextToken in interface antlr.TokenStream

Throws:

antlr.TokenStreamException

reset

public final void reset()

setInput

public final void setInput(Input input)

setSource

public final void setSource(java.io.Reader source)

setSource

public final void setSource(java.io.Reader source,
             java.lang.String name)

ungetToken

public final void ungetToken()