src/antlr/TokenStreamHiddenTokenFilter.cpp

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 *
 * $Id: TokenStreamHiddenTokenFilter.cpp,v 1.1 2005-12-02 15:38:02 tim Exp $
 */
#include "antlr/TokenStreamHiddenTokenFilter.hpp"
#include "antlr/CommonHiddenStreamToken.hpp"

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/**This object filters a token stream coming from a lexer
 * or another TokenStream so that only certain token channels
 * get transmitted to the parser.
 *
 * Any of the channels can be filtered off as "hidden" channels whose
 * tokens can be accessed from the parser.
 */

TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input)
: TokenStreamBasicFilter(input)
{
}

void TokenStreamHiddenTokenFilter::consume()
{
        nextMonitoredToken = input->nextToken();
}

void TokenStreamHiddenTokenFilter::consumeFirst()
{
        consume();

        // Handle situation where hidden or discarded tokens
        // appear first in input stream
        RefToken p;
        // while hidden or discarded scarf tokens
        while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
                if ( hideMask.member(LA(1)->getType()) ) {
                        if ( !p ) {
                                p = LA(1);
                        }
                        else {
                                static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
                                static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link
                                p = LA(1);
                        }
                        lastHiddenToken = p;
                        if (!firstHidden)
                                firstHidden = p; // record hidden token if first
                }
                consume();
        }
}

BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const
{
        return discardMask;
}

/** Return a ptr to the hidden token appearing immediately after
 *  token t in the input stream.
 */
RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t)
{
        return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter();
}

/** Return a ptr to the hidden token appearing immediately before
 *  token t in the input stream.
 */
RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t)
{
        return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore();
}

BitSet TokenStreamHiddenTokenFilter::getHideMask() const
{
        return hideMask;
}

/** Return the first hidden token if one appears
 *  before any monitored token.
 */
RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken()
{
        return firstHidden;
}

void TokenStreamHiddenTokenFilter::hide(int m)
{
        hideMask.add(m);
}

void TokenStreamHiddenTokenFilter::hide(const BitSet& mask)
{
        hideMask = mask;
}

RefToken TokenStreamHiddenTokenFilter::LA(int)
{
        return nextMonitoredToken;
}

/** Return the next monitored token.
*  Test the token following the monitored token.
*  If following is another monitored token, save it
*  for the next invocation of nextToken (like a single
*  lookahead token) and return it then.
*  If following is unmonitored, nondiscarded (hidden)
*  channel token, add it to the monitored token.
*
*  Note: EOF must be a monitored Token.
*/
RefToken TokenStreamHiddenTokenFilter::nextToken()
{
        // handle an initial condition; don't want to get lookahead
        // token of this splitter until first call to nextToken
        if ( !LA(1) ) {
                consumeFirst();
        }

        // we always consume hidden tokens after monitored, thus,
        // upon entry LA(1) is a monitored token.
        RefToken monitored = LA(1);
        // point to hidden tokens found during last invocation
        static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken);
        lastHiddenToken = nullToken;

        // Look for hidden tokens, hook them into list emanating
        // from the monitored tokens.
        consume();
        RefToken p = monitored;
        // while hidden or discarded scarf tokens
        while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
                if ( hideMask.member(LA(1)->getType()) ) {
                        // attach the hidden token to the monitored in a chain
                        // link forwards
                        static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
                        // link backwards
                        if (p != monitored) { //hidden cannot point to monitored tokens
                                static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p);
                        }
                        p = lastHiddenToken = LA(1);
                }
                consume();
        }
        return monitored;
}

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

Revision:	2469
Committed:	Fri Dec 2 15:38:03 2005 UTC (18 years, 7 months ago) by tim
File size:	4268 byte(s)
Log Message:	End of the Link --> List Return of the Oject-Oriented replace yacc/lex parser with antlr parser
#	User	Rev	Content
1	tim	2469	/* ANTLR Translator Generator
2			* Project led by Terence Parr at http://www.jGuru.com
3			* Software rights: http://www.antlr.org/license.html
4			*
5			* $Id: TokenStreamHiddenTokenFilter.cpp,v 1.1 2005-12-02 15:38:02 tim Exp $
6			*/
7			#include "antlr/TokenStreamHiddenTokenFilter.hpp"
8			#include "antlr/CommonHiddenStreamToken.hpp"
9
10			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
11			namespace antlr {
12			#endif
13
14			/**This object filters a token stream coming from a lexer
15			* or another TokenStream so that only certain token channels
16			* get transmitted to the parser.
17			*
18			* Any of the channels can be filtered off as "hidden" channels whose
19			* tokens can be accessed from the parser.
20			*/
21
22			TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input)
23			: TokenStreamBasicFilter(input)
24			{
25			}
26
27			void TokenStreamHiddenTokenFilter::consume()
28			{
29			nextMonitoredToken = input->nextToken();
30			}
31
32			void TokenStreamHiddenTokenFilter::consumeFirst()
33			{
34			consume();
35
36			// Handle situation where hidden or discarded tokens
37			// appear first in input stream
38			RefToken p;
39			// while hidden or discarded scarf tokens
40			while ( hideMask.member(LA(1)->getType()) \|\| discardMask.member(LA(1)->getType()) ) {
41			if ( hideMask.member(LA(1)->getType()) ) {
42			if ( !p ) {
43			p = LA(1);
44			}
45			else {
46			static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
47			static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link
48			p = LA(1);
49			}
50			lastHiddenToken = p;
51			if (!firstHidden)
52			firstHidden = p; // record hidden token if first
53			}
54			consume();
55			}
56			}
57
58			BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const
59			{
60			return discardMask;
61			}
62
63			/** Return a ptr to the hidden token appearing immediately after
64			* token t in the input stream.
65			*/
66			RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t)
67			{
68			return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter();
69			}
70
71			/** Return a ptr to the hidden token appearing immediately before
72			* token t in the input stream.
73			*/
74			RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t)
75			{
76			return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore();
77			}
78
79			BitSet TokenStreamHiddenTokenFilter::getHideMask() const
80			{
81			return hideMask;
82			}
83
84			/** Return the first hidden token if one appears
85			* before any monitored token.
86			*/
87			RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken()
88			{
89			return firstHidden;
90			}
91
92			void TokenStreamHiddenTokenFilter::hide(int m)
93			{
94			hideMask.add(m);
95			}
96
97			void TokenStreamHiddenTokenFilter::hide(const BitSet& mask)
98			{
99			hideMask = mask;
100			}
101
102			RefToken TokenStreamHiddenTokenFilter::LA(int)
103			{
104			return nextMonitoredToken;
105			}
106
107			/** Return the next monitored token.
108			* Test the token following the monitored token.
109			* If following is another monitored token, save it
110			* for the next invocation of nextToken (like a single
111			* lookahead token) and return it then.
112			* If following is unmonitored, nondiscarded (hidden)
113			* channel token, add it to the monitored token.
114			*
115			* Note: EOF must be a monitored Token.
116			*/
117			RefToken TokenStreamHiddenTokenFilter::nextToken()
118			{
119			// handle an initial condition; don't want to get lookahead
120			// token of this splitter until first call to nextToken
121			if ( !LA(1) ) {
122			consumeFirst();
123			}
124
125			// we always consume hidden tokens after monitored, thus,
126			// upon entry LA(1) is a monitored token.
127			RefToken monitored = LA(1);
128			// point to hidden tokens found during last invocation
129			static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken);
130			lastHiddenToken = nullToken;
131
132			// Look for hidden tokens, hook them into list emanating
133			// from the monitored tokens.
134			consume();
135			RefToken p = monitored;
136			// while hidden or discarded scarf tokens
137			while ( hideMask.member(LA(1)->getType()) \|\| discardMask.member(LA(1)->getType()) ) {
138			if ( hideMask.member(LA(1)->getType()) ) {
139			// attach the hidden token to the monitored in a chain
140			// link forwards
141			static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
142			// link backwards
143			if (p != monitored) { //hidden cannot point to monitored tokens
144			static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p);
145			}
146			p = lastHiddenToken = LA(1);
147			}
148			consume();
149			}
150			return monitored;
151			}
152
153			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
154			}
155			#endif
156