org.jruby.lexer.yacc

Class RubyYaccLexer

java.lang.Object

org.jruby.lexer.yacc.RubyYaccLexer

public class RubyYaccLexer
extends java.lang.Object

This is a port of the MRI lexer to Java it is compatible to Ruby 1.8.1.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	RubyYaccLexer.Keyword
static class	RubyYaccLexer.LexState

Field Summary

Fields

Modifier and Type	Field and Description
static org.jcodings.Encoding	ASCII8BIT_ENCODING
private static org.jruby.util.ByteList	BEGIN_DOC_MARKER
private StackState	cmdArgumentState
boolean	commandStart
private StackState	conditionState
private org.jcodings.Encoding	encoding
private static org.joni.Regex	encodingRegexp
private static java.lang.String	encodingString
private static org.jruby.util.ByteList	END_DOC_MARKER
private static org.jruby.util.ByteList	END_MARKER
(package private) static int	EOF
private boolean	isOneEight
private boolean	isTwoZero
private RubyYaccLexer.LexState	last_state
private int	leftParenBegin
private RubyYaccLexer.LexState	lex_state
private StrTerm	lex_strterm
private static org.joni.Regex	magicRegexp
private static java.lang.String	magicString
private static java.util.HashMap<java.lang.String,RubyYaccLexer.Keyword>	map
private byte[]	mbcBuf
private int	parenNest
private ParserSupport	parserSupport
private LexerSource	src
private static int	str_dquote
private static int	str_dsym
(package private) static int	STR_FUNC_ESCAPE
(package private) static int	STR_FUNC_EXPAND
(package private) static int	STR_FUNC_INDENT
(package private) static int	STR_FUNC_QWORDS
(package private) static int	STR_FUNC_REGEXP
(package private) static int	STR_FUNC_SYMBOL
private static int	str_regexp
private static int	str_squote
private static int	str_ssym
private static int	str_xquote
private int	token
private java.lang.StringBuilder	tokenBuffer
static org.jcodings.Encoding	USASCII_ENCODING
static org.jcodings.Encoding	UTF8_ENCODING
private IRubyWarnings	warnings
(package private) java.lang.Object	yaccValue

Constructor Summary

Constructors

Constructor and Description
RubyYaccLexer()
RubyYaccLexer(boolean isOneEight)

Method Summary

Modifier and Type	Method and Description
private void	ambiguousOperator(java.lang.String op, java.lang.String syn)
private int	ampersand(boolean spaceSeen)
private void	arg_ambiguous()
private int	at()
private int	backtick(boolean commandState)
private int	bang()
private int	caret()
private int	colon(boolean spaceSeen)
private int	comma(int c)
StrNode	createStrNode(ISourcePosition position, org.jruby.util.ByteList buffer, int flags)
private void	detectUTF8BOM()
private void	determineExpressionState()
private int	doKeyword(RubyYaccLexer.LexState state)
private int	dollar()
private int	dot()
private int	doubleQuote()
StackState	getCmdArgumentState()
StackState	getConditionState()
java.lang.String	getCurrentLine()
org.jcodings.Encoding	getEncoding()
private int	getFloatToken(java.lang.String number)
private int	getIdentifier(int first)
private java.lang.Object	getInteger(java.lang.String value, int radix)
static RubyYaccLexer.Keyword	getKeyword(java.lang.String str)
int	getLeftParenBegin()
private int	getNumberToken(java.lang.String number, boolean isFloat, int nondigit)
ISourcePosition	getPosition()
ISourcePosition	getPosition(ISourcePosition startPosition) Get position information for Token/Node that follows node represented by startPosition and current lexer location.
StrTerm	getStrTerm()
java.lang.StringBuilder	getTokenBuffer()
private int	greaterThan()
protected void	handleFileEncodingComment(org.jruby.util.ByteList encodingLine)
private int	hereDocumentIdentifier()
private int	identifier(int c, boolean commandState)
private int	identifierToken(int result, java.lang.String value)
int	incrementParenNest()
private boolean	isARG()
private boolean	isBEG()
private boolean	isEND()
boolean	isGlobalCharPunct(int c)
(package private) static boolean	isHexChar(int c)
boolean	isIdentifierChar(int c) This is a valid character for an identifier?
private boolean	isLabelPossible(boolean commandState)
protected boolean	isMultiByteChar(int c) Is this a multibyte character from a multibyte encoding?
private boolean	isNext_identchar()
(package private) static boolean	isOctChar(int c)
boolean	isOneEight()
private boolean	isSpaceArg(int c, boolean spaceSeen)
private int	leftBracket(boolean spaceSeen)
private int	leftCurly()
private int	leftParen(boolean spaceSeen)
private int	lessThan(boolean spaceSeen)
private int	magicCommentMarker(org.jruby.util.ByteList str, int begin)
private boolean	magicCommentSpecialChar(char c)
private int	minus(boolean spaceSeen)
private java.lang.Object	newBignumNode(java.lang.String value, int radix)
private java.lang.Object	newFixnumNode(java.lang.String value, int radix)
int	nextToken()
protected boolean	parseMagicComment(org.jruby.util.ByteList magicLine)
private int	parseNumber(int c) Parse a number from the input stream.
private int	parseQuote(int c) What type/kind of quote are we dealing with?
private int	percent(boolean spaceSeen)
private int	pipe()
private int	plus(boolean spaceSeen)
private void	printState()
private void	printToken(int token)
private int	questionMark()
protected int	readComment() Read a comment up to end of line.
int	readEscape()
private void	readUTF8EscapeIntoBuffer(int codepoint, org.jruby.util.ByteList buffer, boolean stringLiteral)
int	readUTFEscape(org.jruby.util.ByteList buffer, boolean stringLiteral, boolean symbolLiteral)
void	readUTFEscapeRegexpLiteral(org.jruby.util.ByteList buffer)
void	reset()
void	resetStacks()
private int	rightBracket()
private int	rightCurly()
private int	rightParen()
private int	scanHex(int count, boolean strict, java.lang.String errorMessage) Read up to count hexadecimal digits.
private char	scanHexLiteral(org.jruby.util.ByteList buffer, int count, boolean strict, java.lang.String errorMessage) Read up to count hexadecimal digits and store those digits in a token buffer.
private char	scanOct(int count)
private void	setEncoding(org.jruby.util.ByteList name)
void	setEncoding(org.jcodings.Encoding encoding)
void	setLeftParenBegin(int value)
void	setParserSupport(ParserSupport parserSupport) Parse must pass its support object for some check at bottom of yylex().
void	setSource(LexerSource source) Allow the parser to set the source for its lexer.
void	setState(RubyYaccLexer.LexState state)
void	setStrTerm(StrTerm strterm)
void	setValue(java.lang.Object yaccValue)
void	setWarnings(IRubyWarnings warnings)
private int	singleQuote()
private int	slash(boolean spaceSeen)
private int	star(boolean spaceSeen)
private int	tilde()
int	token() Last token read from the lexer at the end of a call to yylex()
int	tokenAddMBC(int codepoint, org.jruby.util.ByteList buffer)
void	tokenAddMBCFromSrc(int c, org.jruby.util.ByteList buffer)
java.lang.Object	value() Value of last token (if it is a token which has a value).
private void	warn_balanced(int c, boolean spaceSeen, java.lang.String op, java.lang.String syn)
private int	yylex() Returns the next token.
private int	yylex2()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

UTF8_ENCODING

public static final org.jcodings.Encoding UTF8_ENCODING

USASCII_ENCODING

public static final org.jcodings.Encoding USASCII_ENCODING

ASCII8BIT_ENCODING

public static final org.jcodings.Encoding ASCII8BIT_ENCODING

END_MARKER

private static org.jruby.util.ByteList END_MARKER

BEGIN_DOC_MARKER

private static org.jruby.util.ByteList BEGIN_DOC_MARKER

END_DOC_MARKER

private static org.jruby.util.ByteList END_DOC_MARKER

map

private static final java.util.HashMap<java.lang.String,RubyYaccLexer.Keyword> map

encoding

private org.jcodings.Encoding encoding

token

private int token

yaccValue

java.lang.Object yaccValue

src

private LexerSource src

parserSupport

private ParserSupport parserSupport

warnings

private IRubyWarnings warnings

lex_state

private RubyYaccLexer.LexState lex_state

last_state

private RubyYaccLexer.LexState last_state

tokenBuffer

private java.lang.StringBuilder tokenBuffer

conditionState

private StackState conditionState

cmdArgumentState

private StackState cmdArgumentState

lex_strterm

private StrTerm lex_strterm

commandStart

public boolean commandStart

EOF

static final int EOF

See Also:

Constant Field Values

STR_FUNC_ESCAPE

static final int STR_FUNC_ESCAPE

See Also:

Constant Field Values

STR_FUNC_EXPAND

static final int STR_FUNC_EXPAND

See Also:

Constant Field Values

STR_FUNC_REGEXP

static final int STR_FUNC_REGEXP

See Also:

Constant Field Values

STR_FUNC_QWORDS

static final int STR_FUNC_QWORDS

See Also:

Constant Field Values

STR_FUNC_SYMBOL

static final int STR_FUNC_SYMBOL

See Also:

Constant Field Values

STR_FUNC_INDENT

static final int STR_FUNC_INDENT

See Also:

Constant Field Values

str_squote

private static final int str_squote

See Also:

Constant Field Values

str_dquote

private static final int str_dquote

See Also:

Constant Field Values

str_xquote

private static final int str_xquote

See Also:

Constant Field Values

str_regexp

private static final int str_regexp

See Also:

Constant Field Values

str_ssym

private static final int str_ssym

See Also:

Constant Field Values

str_dsym

private static final int str_dsym

See Also:

Constant Field Values

isOneEight

private boolean isOneEight

isTwoZero

private boolean isTwoZero

parenNest

private int parenNest

leftParenBegin

private int leftParenBegin

magicString

private static final java.lang.String magicString

See Also:

Constant Field Values

magicRegexp

private static final org.joni.Regex magicRegexp

encodingString

private static final java.lang.String encodingString

See Also:

Constant Field Values

encodingRegexp

private static final org.joni.Regex encodingRegexp

mbcBuf

private byte[] mbcBuf

Constructor Detail

RubyYaccLexer

public RubyYaccLexer()

RubyYaccLexer

public RubyYaccLexer(boolean isOneEight)

Method Detail

getEncoding

public org.jcodings.Encoding getEncoding()

getFloatToken

private int getFloatToken(java.lang.String number)

newBignumNode

private java.lang.Object newBignumNode(java.lang.String value,
                                       int radix)

newFixnumNode

private java.lang.Object newFixnumNode(java.lang.String value,
                                       int radix)
                                throws java.lang.NumberFormatException

Throws:

java.lang.NumberFormatException

ambiguousOperator

private void ambiguousOperator(java.lang.String op,
                               java.lang.String syn)

warn_balanced

private void warn_balanced(int c,
                           boolean spaceSeen,
                           java.lang.String op,
                           java.lang.String syn)

detectUTF8BOM

private void detectUTF8BOM()
                    throws java.io.IOException

Throws:

java.io.IOException

getKeyword

public static RubyYaccLexer.Keyword getKeyword(java.lang.String str)

incrementParenNest

public int incrementParenNest()

getLeftParenBegin

public int getLeftParenBegin()

setLeftParenBegin

public void setLeftParenBegin(int value)

reset

public final void reset()

nextToken

public int nextToken()
              throws java.io.IOException

Throws:

java.io.IOException

token

public int token()

Last token read from the lexer at the end of a call to yylex()

Returns:

last token read

getTokenBuffer

public java.lang.StringBuilder getTokenBuffer()

value

public java.lang.Object value()

Value of last token (if it is a token which has a value).

Returns:

value of last value-laden token

getPosition

public ISourcePosition getPosition(ISourcePosition startPosition)

Get position information for Token/Node that follows node represented by startPosition and current lexer location.

Parameters:

startPosition - previous node/token

inclusive - include previous node into position information of current node

Returns:

a new position

getPosition

public ISourcePosition getPosition()

getCurrentLine

public java.lang.String getCurrentLine()

setParserSupport

public void setParserSupport(ParserSupport parserSupport)

Parse must pass its support object for some check at bottom of yylex(). Ruby does it this way as well (i.e. a little parsing logic in the lexer).

Parameters:

parserSupport -

setEncoding

private void setEncoding(org.jruby.util.ByteList name)

setEncoding

public void setEncoding(org.jcodings.Encoding encoding)

setSource

public void setSource(LexerSource source)

Allow the parser to set the source for its lexer.

Parameters:

source - where the lexer gets raw data

getStrTerm

public StrTerm getStrTerm()

setStrTerm

public void setStrTerm(StrTerm strterm)

resetStacks

public void resetStacks()

setWarnings

public void setWarnings(IRubyWarnings warnings)

printState

private void printState()

setState

public void setState(RubyYaccLexer.LexState state)

getCmdArgumentState

public StackState getCmdArgumentState()

isOneEight

public boolean isOneEight()

getConditionState

public StackState getConditionState()

setValue

public void setValue(java.lang.Object yaccValue)

isNext_identchar

private boolean isNext_identchar()
                          throws java.io.IOException

Throws:

java.io.IOException

isBEG

private boolean isBEG()

isEND

private boolean isEND()

isARG

private boolean isARG()

isLabelPossible

private boolean isLabelPossible(boolean commandState)

isSpaceArg

private boolean isSpaceArg(int c,
                           boolean spaceSeen)

determineExpressionState

private void determineExpressionState()

getInteger

private java.lang.Object getInteger(java.lang.String value,
                                    int radix)

isHexChar

static boolean isHexChar(int c)

Parameters:

c - the character to test

Returns:

true if character is a hex value (0-9a-f)

isOctChar

static boolean isOctChar(int c)

Parameters:

c - the character to test

Returns:

true if character is an octal value (0-7)

isIdentifierChar

public boolean isIdentifierChar(int c)

This is a valid character for an identifier?

Parameters:

c - is character to be compared

Returns:

whether c is an identifier or not mri: is_identchar

isMultiByteChar

protected boolean isMultiByteChar(int c)

Is this a multibyte character from a multibyte encoding?

Parameters:

c -

Returns:

whether c is an multibyte char or not

createStrNode

public StrNode createStrNode(ISourcePosition position,
                             org.jruby.util.ByteList buffer,
                             int flags)

parseQuote

private int parseQuote(int c)
                throws java.io.IOException

What type/kind of quote are we dealing with?

Parameters:

c - first character the the quote construct

Returns:

a token that specifies the quote type

Throws:

java.io.IOException

hereDocumentIdentifier

private int hereDocumentIdentifier()
                            throws java.io.IOException

Throws:

java.io.IOException

arg_ambiguous

private void arg_ambiguous()

magicCommentMarker

private int magicCommentMarker(org.jruby.util.ByteList str,
                               int begin)

magicCommentSpecialChar

private boolean magicCommentSpecialChar(char c)

parseMagicComment

protected boolean parseMagicComment(org.jruby.util.ByteList magicLine)
                             throws java.io.IOException

Throws:

java.io.IOException

handleFileEncodingComment

protected void handleFileEncodingComment(org.jruby.util.ByteList encodingLine)
                                  throws java.io.IOException

Throws:

java.io.IOException

readComment

protected int readComment()
                   throws java.io.IOException

Read a comment up to end of line.

Returns:

something or eof value

Throws:

java.io.IOException

printToken

private void printToken(int token)

yylex2

private int yylex2()
            throws java.io.IOException

Throws:

java.io.IOException

yylex

private int yylex()
           throws java.io.IOException

Returns the next token. Also sets yyVal is needed.

Returns:

Description of the Returned Value

Throws:

java.io.IOException

identifierToken

private int identifierToken(int result,
                            java.lang.String value)

getIdentifier

private int getIdentifier(int first)
                   throws java.io.IOException

Throws:

java.io.IOException

ampersand

private int ampersand(boolean spaceSeen)
               throws java.io.IOException

Throws:

java.io.IOException

at

private int at()
        throws java.io.IOException

Throws:

java.io.IOException

backtick

private int backtick(boolean commandState)
              throws java.io.IOException

Throws:

java.io.IOException

bang

private int bang()
          throws java.io.IOException

Throws:

java.io.IOException

caret

private int caret()
           throws java.io.IOException

Throws:

java.io.IOException

colon

private int colon(boolean spaceSeen)
           throws java.io.IOException

Throws:

java.io.IOException

comma

private int comma(int c)
           throws java.io.IOException

Throws:

java.io.IOException

doKeyword

private int doKeyword(RubyYaccLexer.LexState state)

dollar

private int dollar()
            throws java.io.IOException

Throws:

java.io.IOException

isGlobalCharPunct

public boolean isGlobalCharPunct(int c)

dot

private int dot()
         throws java.io.IOException

Throws:

java.io.IOException

doubleQuote

private int doubleQuote()
                 throws java.io.IOException

Throws:

java.io.IOException

greaterThan

private int greaterThan()
                 throws java.io.IOException

Throws:

java.io.IOException

identifier

private int identifier(int c,
                       boolean commandState)
                throws java.io.IOException

Throws:

java.io.IOException

leftBracket

private int leftBracket(boolean spaceSeen)
                 throws java.io.IOException

Throws:

java.io.IOException

leftCurly

private int leftCurly()

leftParen

private int leftParen(boolean spaceSeen)
               throws java.io.IOException

Throws:

java.io.IOException

lessThan

private int lessThan(boolean spaceSeen)
              throws java.io.IOException

Throws:

java.io.IOException

minus

private int minus(boolean spaceSeen)
           throws java.io.IOException

Throws:

java.io.IOException

percent

private int percent(boolean spaceSeen)
             throws java.io.IOException

Throws:

java.io.IOException

pipe

private int pipe()
          throws java.io.IOException

Throws:

java.io.IOException

plus

private int plus(boolean spaceSeen)
          throws java.io.IOException

Throws:

java.io.IOException

questionMark

private int questionMark()
                  throws java.io.IOException

Throws:

java.io.IOException

rightBracket

private int rightBracket()

rightCurly

private int rightCurly()

rightParen

private int rightParen()

singleQuote

private int singleQuote()
                 throws java.io.IOException

Throws:

java.io.IOException

slash

private int slash(boolean spaceSeen)
           throws java.io.IOException

Throws:

java.io.IOException

star

private int star(boolean spaceSeen)
          throws java.io.IOException

Throws:

java.io.IOException

tilde

private int tilde()
           throws java.io.IOException

Throws:

java.io.IOException

parseNumber

private int parseNumber(int c)
                 throws java.io.IOException

Parse a number from the input stream.

Parameters:

c - The first character of the number.

Returns:

A int constant wich represents a token.

Throws:

java.io.IOException

getNumberToken

private int getNumberToken(java.lang.String number,
                           boolean isFloat,
                           int nondigit)

readUTFEscapeRegexpLiteral

public void readUTFEscapeRegexpLiteral(org.jruby.util.ByteList buffer)
                                throws java.io.IOException

Throws:

java.io.IOException

tokenAddMBC

public int tokenAddMBC(int codepoint,
                       org.jruby.util.ByteList buffer)

tokenAddMBCFromSrc

public void tokenAddMBCFromSrc(int c,
                               org.jruby.util.ByteList buffer)
                        throws java.io.IOException

Throws:

java.io.IOException

readUTFEscape

public int readUTFEscape(org.jruby.util.ByteList buffer,
                         boolean stringLiteral,
                         boolean symbolLiteral)
                  throws java.io.IOException

Throws:

java.io.IOException

readUTF8EscapeIntoBuffer

private void readUTF8EscapeIntoBuffer(int codepoint,
                                      org.jruby.util.ByteList buffer,
                                      boolean stringLiteral)

readEscape

public int readEscape()
               throws java.io.IOException

Throws:

java.io.IOException

scanHexLiteral

private char scanHexLiteral(org.jruby.util.ByteList buffer,
                            int count,
                            boolean strict,
                            java.lang.String errorMessage)
                     throws java.io.IOException

Read up to count hexadecimal digits and store those digits in a token buffer. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown. This will also return the codepoint as a value so codepoint ranges can be checked.

Throws:

java.io.IOException

scanHex

private int scanHex(int count,
                    boolean strict,
                    java.lang.String errorMessage)
             throws java.io.IOException

Read up to count hexadecimal digits. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown.

Throws:

java.io.IOException

scanOct

private char scanOct(int count)
              throws java.io.IOException

Throws:

java.io.IOException