Superclass of generated lexers. More...

#include <CharScanner.hpp>

Inheritance diagram for antlr::CharScanner:

Classes
class	Tracer
	Tracer class, used when -traceLexer is passed to antlr. More...

Public Member Functions
	CharScanner (InputBuffer &cb, bool case_sensitive)

	CharScanner (InputBuffer *cb, bool case_sensitive)

	CharScanner (const LexerSharedInputState &state, bool case_sensitive)

virtual int	LA (unsigned int i)

virtual void	append (char c)

virtual void	append (const std ::string &s)

virtual void	commit ()

virtual void	recover (const RecognitionException &, const BitSet &tokenSet)
	called by the generated lexer to do error recovery, override to customize the behaviour.

virtual void	consume ()

virtual void	consumeUntil (int c)
	Consume chars until one matches the given char.

virtual void	consumeUntil (const BitSet &set)
	Consume chars until one matches the given set.

virtual unsigned int	mark ()
	Mark the current position and return a id for it.

virtual void	rewind (unsigned int pos)
	Rewind the scanner to a previously marked position.

virtual void	match (int c)
	See if input contains character 'c' throw MismatchedCharException if not.

virtual void	match (const BitSet &b)
	See if input contains element from bitset b throw MismatchedCharException if not.

virtual void	match (const char *s)
	See if input contains string 's' throw MismatchedCharException if not.

virtual void	match (const std ::string &s)
	See if input contains string 's' throw MismatchedCharException if not.

virtual void	matchNot (int c)
	See if input does not contain character 'c' throw MismatchedCharException if not.

virtual void	matchRange (int c1, int c2)
	See if input contains character in range c1-c2 throw MismatchedCharException if not.

virtual bool	getCaseSensitive () const

virtual void	setCaseSensitive (bool t)

virtual bool	getCaseSensitiveLiterals () const =0

virtual int	getLine () const
	Get the line the scanner currently is in (starts at 1)

virtual void	setLine (int l)
	set the line number

virtual int	getColumn () const
	Get the column the scanner currently is in (starts at 1)

virtual void	setColumn (int c)
	set the column number

virtual const std::string &	getFilename () const
	get the filename for the file currently used

virtual void	setFilename (const std ::string &f)
	Set the filename the scanner is using (used in error messages)

virtual bool	getCommitToPath () const

virtual void	setCommitToPath (bool commit)

virtual const std::string &	getText () const
	return a copy of the current text buffer

virtual void	setText (const std ::string &s)

virtual void	resetText ()

virtual RefToken	getTokenObject () const

virtual void	newline ()
	Used to keep track of line breaks, needs to be called from within generated lexers when a \r is encountered.

virtual void	tab ()
	Advance the current column number by an appropriate amount according to the tabsize.

int	setTabsize (int size)
	set the tabsize. Returns the old tabsize

int	getTabSize () const
	Return the tabsize used by the scanner.

virtual void	reportError (const RecognitionException &e)
	Report exception errors caught in nextToken()

virtual void	reportError (const std ::string &s)
	Parser error-reporting function can be overridden in subclass.

virtual void	reportWarning (const std ::string &s)
	Parser warning-reporting function can be overridden in subclass.

virtual InputBuffer &	getInputBuffer ()

virtual LexerSharedInputState	getInputState ()

virtual void	setInputState (LexerSharedInputState state)
	set the input state for the lexer.

virtual void	setTokenObjectFactory (factory_type factory)
	Set the factory for created tokens.

virtual int	testLiteralsTable (int ttype) const
	Test the token text against the literals table Override this method to perform a different literals test.

virtual int	testLiteralsTable (const std ::string &txt, int ttype) const
	Test the text passed in against the literals table Override this method to perform a different literals test This is used primarily when you want to test a portion of a token.

virtual int	toLower (int c) const
	Override this method to get more specific case handling.

virtual void	uponEOF ()
	This method is called by YourLexer::nextToken() when the lexer has hit EOF condition.

virtual void	traceIndent ()
	Methods used to change tracing behavior.

virtual void	traceIn (const char *rname)

virtual void	traceOut (const char *rname)

Public Member Functions inherited from antlr::TokenStream
virtual RefToken	nextToken ()=0

Static Public Attributes
static const int	EOF_CHAR = EOF

Protected Types
typedef RefToken(*	factory_type) ()

Protected Member Functions
virtual RefToken	makeToken (int t)
	Create a new RefToken of type t.

Protected Attributes
std::string	text
	Text of current token flag indicating wether consume saves characters.

bool	saveConsumedInput

factory_type	tokenFactory
	Factory for tokens.

bool	caseSensitive
	Is this lexer case sensitive.

std ::map< std ::string, int, CharScannerLiteralsLess >	literals

RefToken	_returnToken
	used to return tokens w/o using return val

LexerSharedInputState	inputState
	Input state, gives access to input stream, shared among different lexers.

bool	commitToPath
	Used during filter mode to indicate that path is desired.

int	tabsize
	tab size the scanner uses.

int	traceDepth

Detailed Description

Superclass of generated lexers.

Definition at line 93 of file CharScanner.hpp.

Member Typedef Documentation

◆ factory_type

typedef RefToken(* antlr::CharScanner::factory_type) ()

protected

Definition at line 95 of file CharScanner.hpp.

Constructor & Destructor Documentation

◆ CharScanner() [1/3]

antlr::CharScanner::CharScanner	(	InputBuffer &	cb,
		bool	case_sensitive )

Definition at line 18 of file CharScanner.cpp.

◆ CharScanner() [2/3]

antlr::CharScanner::CharScanner	(	InputBuffer *	cb,
		bool	case_sensitive )

Definition at line 30 of file CharScanner.cpp.

◆ CharScanner() [3/3]

antlr::CharScanner::CharScanner	(	const LexerSharedInputState &	state,
		bool	case_sensitive )

Definition at line 42 of file CharScanner.cpp.

◆ ~CharScanner()

virtual antlr::CharScanner::~CharScanner ( )

inlinevirtual

Definition at line 101 of file CharScanner.hpp.

Member Function Documentation

◆ append() [1/2]

virtual void antlr::CharScanner::append ( char c )

inlinevirtual

Definition at line 107 of file CharScanner.hpp.

◆ append() [2/2]

virtual void antlr::CharScanner::append ( const std ::string & s )

inlinevirtual

Definition at line 120 of file CharScanner.hpp.

◆ commit()

virtual void antlr::CharScanner::commit ( )

inlinevirtual

Definition at line 126 of file CharScanner.hpp.

◆ consume()

virtual void antlr::CharScanner::consume ( )

inlinevirtual

Definition at line 140 of file CharScanner.hpp.

◆ consumeUntil() [1/2]

virtual void antlr::CharScanner::consumeUntil ( const BitSet & set )

inlinevirtual

Consume chars until one matches the given set.

Definition at line 178 of file CharScanner.hpp.

◆ consumeUntil() [2/2]

virtual void antlr::CharScanner::consumeUntil ( int c )

inlinevirtual

Consume chars until one matches the given char.

Definition at line 166 of file CharScanner.hpp.

◆ getCaseSensitive()

virtual bool antlr::CharScanner::getCaseSensitive ( ) const

inlinevirtual

Definition at line 280 of file CharScanner.hpp.

◆ getColumn()

virtual int antlr::CharScanner::getColumn ( ) const

inlinevirtual

Get the column the scanner currently is in (starts at 1)

Definition at line 305 of file CharScanner.hpp.

◆ getCommitToPath()

virtual bool antlr::CharScanner::getCommitToPath ( ) const

inlinevirtual

Definition at line 326 of file CharScanner.hpp.

◆ getFilename()

virtual const std::string & antlr::CharScanner::getFilename ( ) const

inlinevirtual

get the filename for the file currently used

Definition at line 316 of file CharScanner.hpp.

◆ getInputBuffer()

virtual InputBuffer & antlr::CharScanner::getInputBuffer ( )

inlinevirtual

Definition at line 400 of file CharScanner.hpp.

◆ getInputState()

virtual LexerSharedInputState antlr::CharScanner::getInputState ( )

inlinevirtual

Definition at line 405 of file CharScanner.hpp.

◆ getLine()

virtual int antlr::CharScanner::getLine ( ) const

inlinevirtual

Get the line the scanner currently is in (starts at 1)

Definition at line 293 of file CharScanner.hpp.

◆ getTabSize()

int antlr::CharScanner::getTabSize ( ) const

inline

Return the tabsize used by the scanner.

Definition at line 386 of file CharScanner.hpp.

◆ getText()

virtual const std::string & antlr::CharScanner::getText ( ) const

inlinevirtual

return a copy of the current text buffer

Definition at line 337 of file CharScanner.hpp.

◆ getTokenObject()

virtual RefToken antlr::CharScanner::getTokenObject ( ) const

inlinevirtual

Definition at line 354 of file CharScanner.hpp.

◆ LA()

int antlr::CharScanner::LA ( unsigned int i )

inlinevirtual

Definition at line 553 of file CharScanner.hpp.

◆ makeToken()

virtual RefToken antlr::CharScanner::makeToken ( int t )

inlineprotectedvirtual

Create a new RefToken of type t.

Definition at line 509 of file CharScanner.hpp.

◆ mark()

virtual unsigned int antlr::CharScanner::mark ( )

inlinevirtual

Mark the current position and return a id for it.

Definition at line 190 of file CharScanner.hpp.

◆ match() [1/4]

virtual void antlr::CharScanner::match ( const BitSet & b )

inlinevirtual

See if input contains element from bitset b throw MismatchedCharException if not.

Definition at line 212 of file CharScanner.hpp.

◆ match() [2/4]

virtual void antlr::CharScanner::match ( const char * s )

inlinevirtual

See if input contains string 's' throw MismatchedCharException if not.

Note: the string cannot match EOF

Definition at line 224 of file CharScanner.hpp.

◆ match() [3/4]

virtual void antlr::CharScanner::match ( const std ::string & s )

inlinevirtual

See if input contains string 's' throw MismatchedCharException if not.

Note: the string cannot match EOF

Definition at line 240 of file CharScanner.hpp.

◆ match() [4/4]

virtual void antlr::CharScanner::match ( int c )

inlinevirtual

See if input contains character 'c' throw MismatchedCharException if not.

Definition at line 201 of file CharScanner.hpp.

◆ matchNot()

virtual void antlr::CharScanner::matchNot ( int c )

inlinevirtual

See if input does not contain character 'c' throw MismatchedCharException if not.

Definition at line 258 of file CharScanner.hpp.

◆ matchRange()

virtual void antlr::CharScanner::matchRange	(	int	c1,
		int	c2 )

inlinevirtual

See if input contains character in range c1-c2 throw MismatchedCharException if not.

Definition at line 270 of file CharScanner.hpp.

◆ newline()

virtual void antlr::CharScanner::newline ( )

inlinevirtual

Used to keep track of line breaks, needs to be called from within generated lexers when a
\r is encountered.

Definition at line 362 of file CharScanner.hpp.

◆ recover()

virtual void antlr::CharScanner::recover	(	const RecognitionException &	,
		const BitSet &	tokenSet )

inlinevirtual

called by the generated lexer to do error recovery, override to customize the behaviour.

Definition at line 134 of file CharScanner.hpp.

◆ reportError() [1/2]

void antlr::CharScanner::reportError ( const RecognitionException & e )

virtual

Report exception errors caught in nextToken()

Definition at line 55 of file CharScanner.cpp.

References antlr::RecognitionException::toString().

◆ reportError() [2/2]

void antlr::CharScanner::reportError ( const std ::string & s )

virtual

Parser error-reporting function can be overridden in subclass.

Definition at line 61 of file CharScanner.cpp.

◆ reportWarning()

void antlr::CharScanner::reportWarning ( const std ::string & s )

virtual

Parser warning-reporting function can be overridden in subclass.

Definition at line 70 of file CharScanner.cpp.

◆ resetText()

virtual void antlr::CharScanner::resetText ( )

inlinevirtual

Definition at line 347 of file CharScanner.hpp.

◆ rewind()

virtual void antlr::CharScanner::rewind ( unsigned int pos )

inlinevirtual

Rewind the scanner to a previously marked position.

Definition at line 195 of file CharScanner.hpp.

◆ setCaseSensitive()

virtual void antlr::CharScanner::setCaseSensitive ( bool t )

inlinevirtual

Definition at line 285 of file CharScanner.hpp.

◆ setColumn()

virtual void antlr::CharScanner::setColumn ( int c )

inlinevirtual

set the column number

Definition at line 310 of file CharScanner.hpp.

◆ setCommitToPath()

virtual void antlr::CharScanner::setCommitToPath ( bool commit )

inlinevirtual

Definition at line 331 of file CharScanner.hpp.

◆ setFilename()

virtual void antlr::CharScanner::setFilename ( const std ::string & f )

inlinevirtual

Set the filename the scanner is using (used in error messages)

Definition at line 321 of file CharScanner.hpp.

◆ setInputState()

virtual void antlr::CharScanner::setInputState ( LexerSharedInputState state )

inlinevirtual

set the input state for the lexer.

Note: state is a reference counted object, hence no reference

Definition at line 412 of file CharScanner.hpp.

◆ setLine()

virtual void antlr::CharScanner::setLine ( int l )

inlinevirtual

set the line number

Definition at line 299 of file CharScanner.hpp.

◆ setTabsize()

int antlr::CharScanner::setTabsize ( int size )

inline

set the tabsize. Returns the old tabsize

Definition at line 379 of file CharScanner.hpp.

◆ setText()

virtual void antlr::CharScanner::setText ( const std ::string & s )

inlinevirtual

Definition at line 342 of file CharScanner.hpp.

◆ setTokenObjectFactory()

virtual void antlr::CharScanner::setTokenObjectFactory ( factory_type factory )

inlinevirtual

Set the factory for created tokens.

Definition at line 418 of file CharScanner.hpp.

◆ tab()

virtual void antlr::CharScanner::tab ( )

inlinevirtual

Advance the current column number by an appropriate amount according to the tabsize.

This method needs to be explicitly called from the lexer rules encountering tabs.

Definition at line 372 of file CharScanner.hpp.

◆ testLiteralsTable() [1/2]

virtual int antlr::CharScanner::testLiteralsTable	(	const std ::string &	txt,
		int	ttype ) const

inlinevirtual

Test the text passed in against the literals table Override this method to perform a different literals test This is used primarily when you want to test a portion of a token.

Definition at line 439 of file CharScanner.hpp.

◆ testLiteralsTable() [2/2]

virtual int antlr::CharScanner::testLiteralsTable ( int ttype ) const

inlinevirtual

Test the token text against the literals table Override this method to perform a different literals test.

Definition at line 426 of file CharScanner.hpp.

◆ toLower()

virtual int antlr::CharScanner::toLower ( int c ) const

inlinevirtual

Override this method to get more specific case handling.

Definition at line 448 of file CharScanner.hpp.

◆ traceIn()

void antlr::CharScanner::traceIn ( const char * rname )

virtual

Definition at line 84 of file CharScanner.cpp.

◆ traceIndent()

void antlr::CharScanner::traceIndent ( )

virtual

Methods used to change tracing behavior.

Definition at line 78 of file CharScanner.cpp.

◆ traceOut()

void antlr::CharScanner::traceOut ( const char * rname )

virtual

Definition at line 92 of file CharScanner.cpp.

◆ uponEOF()

virtual void antlr::CharScanner::uponEOF ( )

inlinevirtual

This method is called by YourLexer::nextToken() when the lexer has hit EOF condition.

EOF is NOT a character. This method is not called if EOF is reached during syntactic predicate evaluation or during evaluation of normal lexical rules, which presumably would be an IOException. This traps the "normal" EOF condition.

uponEOF() is called after the complete evaluation of the previous token and only if your parser asks for another token beyond that last non-EOF token.

You might want to throw token or char stream exceptions like: "Heh, premature eof" or a retry stream exception ("I found the end of this file, go back to referencing file").

Definition at line 471 of file CharScanner.hpp.

Member Data Documentation

◆ _returnToken

RefToken antlr::CharScanner::_returnToken

protected

used to return tokens w/o using return val

Definition at line 495 of file CharScanner.hpp.

◆ caseSensitive

bool antlr::CharScanner::caseSensitive

protected

Is this lexer case sensitive.

Definition at line 492 of file CharScanner.hpp.

◆ commitToPath

bool antlr::CharScanner::commitToPath

protected

Used during filter mode to indicate that path is desired.

A subsequent scan error will report an error as usual if acceptPath=true;

Definition at line 504 of file CharScanner.hpp.

◆ EOF_CHAR

const int antlr::CharScanner::EOF_CHAR = EOF

static

Definition at line 481 of file CharScanner.hpp.

◆ inputState

LexerSharedInputState antlr::CharScanner::inputState

protected

Input state, gives access to input stream, shared among different lexers.

Definition at line 498 of file CharScanner.hpp.

◆ literals

std :: map< std :: string,int,CharScannerLiteralsLess> antlr::CharScanner::literals

protected

Definition at line 493 of file CharScanner.hpp.

◆ saveConsumedInput

bool antlr::CharScanner::saveConsumedInput

protected

Definition at line 490 of file CharScanner.hpp.

◆ tabsize

int antlr::CharScanner::tabsize

protected

tab size the scanner uses.

Definition at line 506 of file CharScanner.hpp.

◆ text

std::string antlr::CharScanner::text

protected

Text of current token flag indicating wether consume saves characters.

Definition at line 488 of file CharScanner.hpp.

◆ tokenFactory

factory_type antlr::CharScanner::tokenFactory

protected

Factory for tokens.

Definition at line 491 of file CharScanner.hpp.

◆ traceDepth

int antlr::CharScanner::traceDepth

protected

Definition at line 539 of file CharScanner.hpp.

The documentation for this class was generated from the following files:

antlr/CharScanner.hpp
antlr/CharScanner.cpp

Classes

Public Member Functions

Static Public Attributes

Protected Types

Protected Member Functions

Protected Attributes

Detailed Description

Member Typedef Documentation

◆ factory_type

Constructor & Destructor Documentation

◆ CharScanner() [1/3]

◆ CharScanner() [2/3]

◆ CharScanner() [3/3]

◆ ~CharScanner()

Member Function Documentation

◆ append() [1/2]

◆ append() [2/2]

◆ commit()

◆ consume()

◆ consumeUntil() [1/2]

◆ consumeUntil() [2/2]

◆ getCaseSensitive()

◆ getColumn()

◆ getCommitToPath()

◆ getFilename()

◆ getInputBuffer()

◆ getInputState()

◆ getLine()

◆ getTabSize()

◆ getText()

◆ getTokenObject()

◆ LA()

◆ makeToken()

◆ mark()

◆ match() [1/4]

◆ match() [2/4]

◆ match() [3/4]

◆ match() [4/4]

◆ matchNot()

◆ matchRange()

◆ newline()

◆ recover()

◆ reportError() [1/2]

◆ reportError() [2/2]

◆ reportWarning()

◆ resetText()

◆ rewind()

◆ setCaseSensitive()

◆ setColumn()

◆ setCommitToPath()

◆ setFilename()

◆ setInputState()

◆ setLine()

◆ setTabsize()

◆ setText()

◆ setTokenObjectFactory()

◆ tab()

◆ testLiteralsTable() [1/2]

◆ testLiteralsTable() [2/2]

◆ toLower()

◆ traceIn()

◆ traceIndent()

◆ traceOut()

◆ uponEOF()

Member Data Documentation

◆ _returnToken

◆ caseSensitive

◆ commitToPath

◆ EOF_CHAR

◆ inputState

◆ literals

◆ saveConsumedInput

◆ tabsize

◆ text

◆ tokenFactory

◆ traceDepth