class TokenStream
extends java.lang.Object
Return a steady stream of tokens.
The returned tokens are one of: LITERAL, '%', FORMAT_MODIFIER, SIMPLE_KEYWORD, COMPOSITE_KEYWORD OPTION, LEFT_PARENTHESIS, and RIGHT_PARENTHESIS.
The '\' character is used as escape. It can be used to escape '_', '%', '(' and '('.
Note that there is no EOS token returned.
| Modifier and Type | Class and Description |
|---|---|
(package private) static class |
TokenStream.TokenizerState |
| Modifier and Type | Field and Description |
|---|---|
(package private) IEscapeUtil |
escapeUtil |
(package private) IEscapeUtil |
optionEscapeUtil |
(package private) java.lang.String |
pattern |
(package private) int |
patternLength |
(package private) int |
pointer |
(package private) TokenStream.TokenizerState |
state |
| Constructor and Description |
|---|
TokenStream(java.lang.String pattern) |
TokenStream(java.lang.String pattern,
IEscapeUtil escapeUtil) |
| Modifier and Type | Method and Description |
|---|---|
private void |
addValuedToken(int type,
java.lang.StringBuffer buf,
java.util.List<Token> tokenList) |
(package private) void |
escape(java.lang.String escapeChars,
java.lang.StringBuffer buf) |
private void |
handleFormatModifierState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf) |
private void |
handleKeywordState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf) |
private void |
handleLiteralState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf) |
private void |
handleRightParenthesisState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf) |
(package private) void |
optionEscape(java.lang.String escapeChars,
java.lang.StringBuffer buf) |
private void |
processOption(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf) |
(package private) java.util.List |
tokenize() |
final java.lang.String pattern
final int patternLength
final IEscapeUtil escapeUtil
final IEscapeUtil optionEscapeUtil
TokenStream.TokenizerState state
int pointer
TokenStream(java.lang.String pattern)
TokenStream(java.lang.String pattern,
IEscapeUtil escapeUtil)
java.util.List tokenize()
throws ScanException
ScanExceptionprivate void handleRightParenthesisState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf)
private void processOption(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf)
throws ScanException
ScanExceptionprivate void handleFormatModifierState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf)
private void handleLiteralState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf)
private void handleKeywordState(char c,
java.util.List<Token> tokenList,
java.lang.StringBuffer buf)
void escape(java.lang.String escapeChars,
java.lang.StringBuffer buf)
void optionEscape(java.lang.String escapeChars,
java.lang.StringBuffer buf)
private void addValuedToken(int type,
java.lang.StringBuffer buf,
java.util.List<Token> tokenList)