src/antlr/TokenStreamRewriteEngine.cpp

#include <antlr/config.hpp>

#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <iostream>
#include <iterator>
#include <sstream>
#include <cassert>

#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>
#include <antlr/TokenStreamRewriteEngine.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

#ifndef NO_STATIC_CONSTS
const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0;
const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100;
#endif

const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";

namespace {

        struct compareOperationIndex {
                typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
                bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
                {
                        return a->getIndex() < b->getIndex();
                }
        };
        struct dumpTokenWithIndex {
                dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
                void operator() ( const RefTokenWithIndex& t ) {
                        out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
                }
                ANTLR_USE_NAMESPACE(std)ostream& out;
        };
}

TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
: stream(upstream)
, index(MIN_TOKEN_INDEX)
, tokens()
, programs()
, discardMask()
{
}

TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
: stream(upstream)
, index(MIN_TOKEN_INDEX)
, tokens(initialSize)
, programs()
, discardMask()
{
}

RefToken TokenStreamRewriteEngine::nextToken( void )
{
        RefTokenWithIndex t;
        // suck tokens until end of stream or we find a non-discarded token
        do {
                t = RefTokenWithIndex(stream.nextToken());
                if ( t )
                {
                        t->setIndex(index);  // what is t's index in list?
                        if ( t->getType() != Token::EOF_TYPE ) {
                                tokens.push_back(t);  // track all tokens except EOF
                        }
                        index++;                        // move to next position
                }
        } while ( t && discardMask.member(t->getType()) );
        return RefToken(t);
}

void TokenStreamRewriteEngine::rollback( const std::string& programName,
                                                                                                          size_t instructionIndex )
{
        program_map::iterator rewrite = programs.find(programName);
        if( rewrite != programs.end() )
        {
                operation_list& prog = rewrite->second;
                operation_list::iterator
                        j = prog.begin(),
                        end = prog.end();

                std::advance(j,instructionIndex);
                if( j != end )
                        prog.erase(j, end);
        }
}

void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
                                                                                                                                 size_t start,
                                                                                                                                 size_t end ) const
{
        token_list::const_iterator s = tokens.begin();
        std::advance( s, start );
        token_list::const_iterator e = s;
        std::advance( e, end-start );
        std::for_each( s, e, tokenToStream(out) );
}

void TokenStreamRewriteEngine::toStream( std::ostream& out,
                                                                                                          const std::string& programName,
                                                                                                          size_t firstToken,
                                                                                                          size_t lastToken ) const
{
        if( tokens.size() == 0 )
                return;

        program_map::const_iterator rewriter = programs.find(programName);

        if ( rewriter == programs.end() )
                return;

        // get the prog and some iterators in it...
        const operation_list& prog = rewriter->second;
        operation_list::const_iterator
                rewriteOpIndex = prog.begin(),
                rewriteOpEnd = prog.end();

        size_t tokenCursor = firstToken;
        // make sure we don't run out of the tokens we have...
        if( lastToken > (tokens.size() - 1) )
                lastToken = tokens.size() - 1;

                while ( tokenCursor <= lastToken )
                {
//                      std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;

                        if( rewriteOpIndex != rewriteOpEnd )
                        {
                                size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
                                while( tokenCursor < up_to_here )
                                        out << tokens[tokenCursor++]->getText();
                        }
                        while ( rewriteOpIndex != rewriteOpEnd &&
                                          tokenCursor == (*rewriteOpIndex)->getIndex() &&
                                          tokenCursor <= lastToken )
                        {
                                tokenCursor = (*rewriteOpIndex)->execute(out);
                                ++rewriteOpIndex;
                        }
                        if( tokenCursor <= lastToken )
                                out << tokens[tokenCursor++]->getText();
                }
        // std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
        // now see if there are operations (append) beyond last token index
        std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
        rewriteOpIndex = rewriteOpEnd;
}

void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
                                                                                                                         size_t start,
                                                                                                                         size_t end ) const
{
        token_list::const_iterator s = tokens.begin();
        std::advance( s, start );
        token_list::const_iterator e = s;
        std::advance( e, end-start );
        std::for_each( s, e, dumpTokenWithIndex(out) );
}

void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
                                                                                                                                                 RewriteOperation* op )
{
        program_map::iterator rewrites = programs.find(programName);
        // check if we got the program already..
        if ( rewrites == programs.end() )
        {
                // no prog make a new one...
                operation_list ops;
                ops.push_back(op);
                programs.insert(std::make_pair(programName,ops));
                return;
        }
        operation_list& prog = rewrites->second;

        if( prog.empty() )
        {
                prog.push_back(op);
                return;
        }

        operation_list::iterator i, end = prog.end();
        i = end;
        --i;
        // if at or beyond last op's index, just append
        if ( op->getIndex() >= (*i)->getIndex() ) {
                prog.push_back(op); // append to list of operations
                return;
        }
        i = prog.begin();

        if( i != end )
        {
                operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
                prog.insert(pos,op);
        }
        else
                prog.push_back(op);
}

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
Revision:	1633
Committed:	Thu Sep 15 13:39:36 2011 UTC (14 years, 1 month ago) by gezelter
File size:	5703 byte(s)
Log Message:	Updated antlr, fixed a clang compilation bug, removed a warning message
#	User	Rev	Content
1	gezelter	1633	#include <antlr/config.hpp>
2
3	tim	770	#include <string>
4			#include <list>
5			#include <vector>
6			#include <map>
7			#include <utility>
8			#include <iostream>
9			#include <iterator>
10			#include <sstream>
11			#include <cassert>
12
13			#include <antlr/TokenStream.hpp>
14			#include <antlr/TokenWithIndex.hpp>
15			#include <antlr/BitSet.hpp>
16			#include <antlr/TokenStreamRewriteEngine.hpp>
17
18			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
19			namespace antlr {
20			#endif
21
22	gezelter	1633	#ifndef NO_STATIC_CONSTS
23			const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0;
24			const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100;
25			#endif
26
27	tim	770	const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";
28
29			namespace {
30
31			struct compareOperationIndex {
32			typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
33			bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
34			{
35			return a->getIndex() < b->getIndex();
36			}
37			};
38			struct dumpTokenWithIndex {
39			dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
40			void operator() ( const RefTokenWithIndex& t ) {
41			out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
42			}
43			ANTLR_USE_NAMESPACE(std)ostream& out;
44			};
45	gezelter	783	}
46	tim	770
47			TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
48			: stream(upstream)
49			, index(MIN_TOKEN_INDEX)
50			, tokens()
51			, programs()
52			, discardMask()
53			{
54			}
55
56			TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
57			: stream(upstream)
58			, index(MIN_TOKEN_INDEX)
59			, tokens(initialSize)
60			, programs()
61			, discardMask()
62			{
63			}
64
65			RefToken TokenStreamRewriteEngine::nextToken( void )
66			{
67			RefTokenWithIndex t;
68			// suck tokens until end of stream or we find a non-discarded token
69			do {
70			t = RefTokenWithIndex(stream.nextToken());
71			if ( t )
72			{
73			t->setIndex(index); // what is t's index in list?
74			if ( t->getType() != Token::EOF_TYPE ) {
75			tokens.push_back(t); // track all tokens except EOF
76			}
77			index++; // move to next position
78			}
79			} while ( t && discardMask.member(t->getType()) );
80			return RefToken(t);
81			}
82
83			void TokenStreamRewriteEngine::rollback( const std::string& programName,
84			size_t instructionIndex )
85			{
86			program_map::iterator rewrite = programs.find(programName);
87			if( rewrite != programs.end() )
88			{
89			operation_list& prog = rewrite->second;
90			operation_list::iterator
91			j = prog.begin(),
92			end = prog.end();
93
94			std::advance(j,instructionIndex);
95			if( j != end )
96			prog.erase(j, end);
97			}
98			}
99
100			void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
101			size_t start,
102			size_t end ) const
103			{
104			token_list::const_iterator s = tokens.begin();
105			std::advance( s, start );
106			token_list::const_iterator e = s;
107			std::advance( e, end-start );
108			std::for_each( s, e, tokenToStream(out) );
109			}
110
111			void TokenStreamRewriteEngine::toStream( std::ostream& out,
112			const std::string& programName,
113			size_t firstToken,
114			size_t lastToken ) const
115			{
116			if( tokens.size() == 0 )
117			return;
118
119			program_map::const_iterator rewriter = programs.find(programName);
120
121			if ( rewriter == programs.end() )
122			return;
123
124			// get the prog and some iterators in it...
125			const operation_list& prog = rewriter->second;
126			operation_list::const_iterator
127			rewriteOpIndex = prog.begin(),
128			rewriteOpEnd = prog.end();
129
130			size_t tokenCursor = firstToken;
131			// make sure we don't run out of the tokens we have...
132			if( lastToken > (tokens.size() - 1) )
133			lastToken = tokens.size() - 1;
134
135			while ( tokenCursor <= lastToken )
136			{
137			// std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;
138
139			if( rewriteOpIndex != rewriteOpEnd )
140			{
141			size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
142			while( tokenCursor < up_to_here )
143			out << tokens[tokenCursor++]->getText();
144			}
145			while ( rewriteOpIndex != rewriteOpEnd &&
146			tokenCursor == (*rewriteOpIndex)->getIndex() &&
147			tokenCursor <= lastToken )
148			{
149			tokenCursor = (*rewriteOpIndex)->execute(out);
150			++rewriteOpIndex;
151			}
152			if( tokenCursor <= lastToken )
153			out << tokens[tokenCursor++]->getText();
154			}
155			// std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
156			// now see if there are operations (append) beyond last token index
157			std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
158			rewriteOpIndex = rewriteOpEnd;
159			}
160
161			void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
162			size_t start,
163			size_t end ) const
164			{
165			token_list::const_iterator s = tokens.begin();
166			std::advance( s, start );
167			token_list::const_iterator e = s;
168			std::advance( e, end-start );
169			std::for_each( s, e, dumpTokenWithIndex(out) );
170			}
171
172			void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
173			RewriteOperation* op )
174			{
175			program_map::iterator rewrites = programs.find(programName);
176			// check if we got the program already..
177			if ( rewrites == programs.end() )
178			{
179			// no prog make a new one...
180			operation_list ops;
181			ops.push_back(op);
182			programs.insert(std::make_pair(programName,ops));
183			return;
184			}
185			operation_list& prog = rewrites->second;
186
187			if( prog.empty() )
188			{
189			prog.push_back(op);
190			return;
191			}
192
193			operation_list::iterator i, end = prog.end();
194			i = end;
195			--i;
196			// if at or beyond last op's index, just append
197			if ( op->getIndex() >= (*i)->getIndex() ) {
198			prog.push_back(op); // append to list of operations
199			return;
200			}
201			i = prog.begin();
202
203			if( i != end )
204			{
205			operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
206			prog.insert(pos,op);
207			}
208			else
209			prog.push_back(op);
210			}
211
212			#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
213			}
214			#endif