src/antlr/TokenStreamHiddenTokenFilter.cpp

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 *
 * $Id: TokenStreamHiddenTokenFilter.cpp,v 1.1 2005-12-02 15:38:02 tim Exp $
 */
#include "antlr/TokenStreamHiddenTokenFilter.hpp"
#include "antlr/CommonHiddenStreamToken.hpp"

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/**This object filters a token stream coming from a lexer
 * or another TokenStream so that only certain token channels
 * get transmitted to the parser.
 *
 * Any of the channels can be filtered off as "hidden" channels whose
 * tokens can be accessed from the parser.
 */

TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input)
: TokenStreamBasicFilter(input)
{
}

void TokenStreamHiddenTokenFilter::consume()
{
        nextMonitoredToken = input->nextToken();
}

void TokenStreamHiddenTokenFilter::consumeFirst()
{
        consume();

        // Handle situation where hidden or discarded tokens
        // appear first in input stream
        RefToken p;
        // while hidden or discarded scarf tokens
        while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
                if ( hideMask.member(LA(1)->getType()) ) {
                        if ( !p ) {
                                p = LA(1);
                        }
                        else {
                                static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
                                static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link
                                p = LA(1);
                        }
                        lastHiddenToken = p;
                        if (!firstHidden)
                                firstHidden = p; // record hidden token if first
                }
                consume();
        }
}

BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const
{
        return discardMask;
}

/** Return a ptr to the hidden token appearing immediately after
 *  token t in the input stream.
 */
RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t)
{
        return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter();
}

/** Return a ptr to the hidden token appearing immediately before
 *  token t in the input stream.
 */
RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t)
{
        return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore();
}

BitSet TokenStreamHiddenTokenFilter::getHideMask() const
{
        return hideMask;
}

/** Return the first hidden token if one appears
 *  before any monitored token.
 */
RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken()
{
        return firstHidden;
}

void TokenStreamHiddenTokenFilter::hide(int m)
{
        hideMask.add(m);
}

void TokenStreamHiddenTokenFilter::hide(const BitSet& mask)
{
        hideMask = mask;
}

RefToken TokenStreamHiddenTokenFilter::LA(int)
{
        return nextMonitoredToken;
}

/** Return the next monitored token.
*  Test the token following the monitored token.
*  If following is another monitored token, save it
*  for the next invocation of nextToken (like a single
*  lookahead token) and return it then.
*  If following is unmonitored, nondiscarded (hidden)
*  channel token, add it to the monitored token.
*
*  Note: EOF must be a monitored Token.
*/
RefToken TokenStreamHiddenTokenFilter::nextToken()
{
        // handle an initial condition; don't want to get lookahead
        // token of this splitter until first call to nextToken
        if ( !LA(1) ) {
                consumeFirst();
        }

        // we always consume hidden tokens after monitored, thus,
        // upon entry LA(1) is a monitored token.
        RefToken monitored = LA(1);
        // point to hidden tokens found during last invocation
        static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken);
        lastHiddenToken = nullToken;

        // Look for hidden tokens, hook them into list emanating
        // from the monitored tokens.
        consume();
        RefToken p = monitored;
        // while hidden or discarded scarf tokens
        while ( hideMask.member(LA(1)->getType()) || discardMask.member(LA(1)->getType()) ) {
                if ( hideMask.member(LA(1)->getType()) ) {
                        // attach the hidden token to the monitored in a chain
                        // link forwards
                        static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
                        // link backwards
                        if (p != monitored) { //hidden cannot point to monitored tokens
                                static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p);
                        }
                        p = lastHiddenToken = LA(1);
                }
                consume();
        }
        return monitored;
}

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

Revision:	2469
Committed:	Fri Dec 2 15:38:03 2005 UTC (18 years, 7 months ago) by tim
File size:	4268 byte(s)
Log Message:	End of the Link --> List Return of the Oject-Oriented replace yacc/lex parser with antlr parser
#	Content
1	/* ANTLR Translator Generator
2	* Project led by Terence Parr at http://www.jGuru.com
3	* Software rights: http://www.antlr.org/license.html
4	*
5	* $Id: TokenStreamHiddenTokenFilter.cpp,v 1.1 2005-12-02 15:38:02 tim Exp $
6	*/
7	#include "antlr/TokenStreamHiddenTokenFilter.hpp"
8	#include "antlr/CommonHiddenStreamToken.hpp"
9
10	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
11	namespace antlr {
12	#endif
13
14	/**This object filters a token stream coming from a lexer
15	* or another TokenStream so that only certain token channels
16	* get transmitted to the parser.
17	*
18	* Any of the channels can be filtered off as "hidden" channels whose
19	* tokens can be accessed from the parser.
20	*/
21
22	TokenStreamHiddenTokenFilter::TokenStreamHiddenTokenFilter(TokenStream& input)
23	: TokenStreamBasicFilter(input)
24	{
25	}
26
27	void TokenStreamHiddenTokenFilter::consume()
28	{
29	nextMonitoredToken = input->nextToken();
30	}
31
32	void TokenStreamHiddenTokenFilter::consumeFirst()
33	{
34	consume();
35
36	// Handle situation where hidden or discarded tokens
37	// appear first in input stream
38	RefToken p;
39	// while hidden or discarded scarf tokens
40	while ( hideMask.member(LA(1)->getType()) \|\| discardMask.member(LA(1)->getType()) ) {
41	if ( hideMask.member(LA(1)->getType()) ) {
42	if ( !p ) {
43	p = LA(1);
44	}
45	else {
46	static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
47	static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p); // double-link
48	p = LA(1);
49	}
50	lastHiddenToken = p;
51	if (!firstHidden)
52	firstHidden = p; // record hidden token if first
53	}
54	consume();
55	}
56	}
57
58	BitSet TokenStreamHiddenTokenFilter::getDiscardMask() const
59	{
60	return discardMask;
61	}
62
63	/** Return a ptr to the hidden token appearing immediately after
64	* token t in the input stream.
65	*/
66	RefToken TokenStreamHiddenTokenFilter::getHiddenAfter(RefToken t)
67	{
68	return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenAfter();
69	}
70
71	/** Return a ptr to the hidden token appearing immediately before
72	* token t in the input stream.
73	*/
74	RefToken TokenStreamHiddenTokenFilter::getHiddenBefore(RefToken t)
75	{
76	return static_cast<CommonHiddenStreamToken*>(t.get())->getHiddenBefore();
77	}
78
79	BitSet TokenStreamHiddenTokenFilter::getHideMask() const
80	{
81	return hideMask;
82	}
83
84	/** Return the first hidden token if one appears
85	* before any monitored token.
86	*/
87	RefToken TokenStreamHiddenTokenFilter::getInitialHiddenToken()
88	{
89	return firstHidden;
90	}
91
92	void TokenStreamHiddenTokenFilter::hide(int m)
93	{
94	hideMask.add(m);
95	}
96
97	void TokenStreamHiddenTokenFilter::hide(const BitSet& mask)
98	{
99	hideMask = mask;
100	}
101
102	RefToken TokenStreamHiddenTokenFilter::LA(int)
103	{
104	return nextMonitoredToken;
105	}
106
107	/** Return the next monitored token.
108	* Test the token following the monitored token.
109	* If following is another monitored token, save it
110	* for the next invocation of nextToken (like a single
111	* lookahead token) and return it then.
112	* If following is unmonitored, nondiscarded (hidden)
113	* channel token, add it to the monitored token.
114	*
115	* Note: EOF must be a monitored Token.
116	*/
117	RefToken TokenStreamHiddenTokenFilter::nextToken()
118	{
119	// handle an initial condition; don't want to get lookahead
120	// token of this splitter until first call to nextToken
121	if ( !LA(1) ) {
122	consumeFirst();
123	}
124
125	// we always consume hidden tokens after monitored, thus,
126	// upon entry LA(1) is a monitored token.
127	RefToken monitored = LA(1);
128	// point to hidden tokens found during last invocation
129	static_cast<CommonHiddenStreamToken*>(monitored.get())->setHiddenBefore(lastHiddenToken);
130	lastHiddenToken = nullToken;
131
132	// Look for hidden tokens, hook them into list emanating
133	// from the monitored tokens.
134	consume();
135	RefToken p = monitored;
136	// while hidden or discarded scarf tokens
137	while ( hideMask.member(LA(1)->getType()) \|\| discardMask.member(LA(1)->getType()) ) {
138	if ( hideMask.member(LA(1)->getType()) ) {
139	// attach the hidden token to the monitored in a chain
140	// link forwards
141	static_cast<CommonHiddenStreamToken*>(p.get())->setHiddenAfter(LA(1));
142	// link backwards
143	if (p != monitored) { //hidden cannot point to monitored tokens
144	static_cast<CommonHiddenStreamToken*>(LA(1).get())->setHiddenBefore(p);
145	}
146	p = lastHiddenToken = LA(1);
147	}
148	consume();
149	}
150	return monitored;
151	}
152
153	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
154	}
155	#endif
156