src/antlr/TokenStreamRewriteEngine.cpp

#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <iostream>
#include <iterator>
#include <sstream>
#include <cassert>

#include <antlr/config.hpp>

#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>
#include <antlr/TokenStreamRewriteEngine.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";

namespace {

        struct compareOperationIndex {
                typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
                bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
                {
                        return a->getIndex() < b->getIndex();
                }
        };
        struct dumpTokenWithIndex {
                dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
                void operator() ( const RefTokenWithIndex& t ) {
                        out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
                }
                ANTLR_USE_NAMESPACE(std)ostream& out;
        };
};

TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
: stream(upstream)
, index(MIN_TOKEN_INDEX)
, tokens()
, programs()
, discardMask()
{
}

TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
: stream(upstream)
, index(MIN_TOKEN_INDEX)
, tokens(initialSize)
, programs()
, discardMask()
{
}

RefToken TokenStreamRewriteEngine::nextToken( void )
{
        RefTokenWithIndex t;
        // suck tokens until end of stream or we find a non-discarded token
        do {
                t = RefTokenWithIndex(stream.nextToken());
                if ( t )
                {
                        t->setIndex(index);  // what is t's index in list?
                        if ( t->getType() != Token::EOF_TYPE ) {
                                tokens.push_back(t);  // track all tokens except EOF
                        }
                        index++;                        // move to next position
                }
        } while ( t && discardMask.member(t->getType()) );
        return RefToken(t);
}

void TokenStreamRewriteEngine::rollback( const std::string& programName,
                                                                                                          size_t instructionIndex )
{
        program_map::iterator rewrite = programs.find(programName);
        if( rewrite != programs.end() )
        {
                operation_list& prog = rewrite->second;
                operation_list::iterator
                        j = prog.begin(),
                        end = prog.end();

                std::advance(j,instructionIndex);
                if( j != end )
                        prog.erase(j, end);
        }
}

void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
                                                                                                                                 size_t start,
                                                                                                                                 size_t end ) const
{
        token_list::const_iterator s = tokens.begin();
        std::advance( s, start );
        token_list::const_iterator e = s;
        std::advance( e, end-start );
        std::for_each( s, e, tokenToStream(out) );
}

void TokenStreamRewriteEngine::toStream( std::ostream& out,
                                                                                                          const std::string& programName,
                                                                                                          size_t firstToken,
                                                                                                          size_t lastToken ) const
{
        if( tokens.size() == 0 )
                return;

        program_map::const_iterator rewriter = programs.find(programName);

        if ( rewriter == programs.end() )
                return;

        // get the prog and some iterators in it...
        const operation_list& prog = rewriter->second;
        operation_list::const_iterator
                rewriteOpIndex = prog.begin(),
                rewriteOpEnd = prog.end();

        size_t tokenCursor = firstToken;
        // make sure we don't run out of the tokens we have...
        if( lastToken > (tokens.size() - 1) )
                lastToken = tokens.size() - 1;

                while ( tokenCursor <= lastToken )
                {
//                      std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;

                        if( rewriteOpIndex != rewriteOpEnd )
                        {
                                size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
                                while( tokenCursor < up_to_here )
                                        out << tokens[tokenCursor++]->getText();
                        }
                        while ( rewriteOpIndex != rewriteOpEnd &&
                                          tokenCursor == (*rewriteOpIndex)->getIndex() &&
                                          tokenCursor <= lastToken )
                        {
                                tokenCursor = (*rewriteOpIndex)->execute(out);
                                ++rewriteOpIndex;
                        }
                        if( tokenCursor <= lastToken )
                                out << tokens[tokenCursor++]->getText();
                }
        // std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
        // now see if there are operations (append) beyond last token index
        std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
        rewriteOpIndex = rewriteOpEnd;
}

void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
                                                                                                                         size_t start,
                                                                                                                         size_t end ) const
{
        token_list::const_iterator s = tokens.begin();
        std::advance( s, start );
        token_list::const_iterator e = s;
        std::advance( e, end-start );
        std::for_each( s, e, dumpTokenWithIndex(out) );
}

void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
                                                                                                                                                 RewriteOperation* op )
{
        program_map::iterator rewrites = programs.find(programName);
        // check if we got the program already..
        if ( rewrites == programs.end() )
        {
                // no prog make a new one...
                operation_list ops;
                ops.push_back(op);
                programs.insert(std::make_pair(programName,ops));
                return;
        }
        operation_list& prog = rewrites->second;

        if( prog.empty() )
        {
                prog.push_back(op);
                return;
        }

        operation_list::iterator i, end = prog.end();
        i = end;
        --i;
        // if at or beyond last op's index, just append
        if ( op->getIndex() >= (*i)->getIndex() ) {
                prog.push_back(op); // append to list of operations
                return;
        }
        i = prog.begin();

        if( i != end )
        {
                operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
                prog.insert(pos,op);
        }
        else
                prog.push_back(op);
}

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
Revision:	770
Committed:	Fri Dec 2 15:38:03 2005 UTC (19 years, 11 months ago) by tim
File size:	5550 byte(s)
Log Message:	End of the Link --> List Return of the Oject-Oriented replace yacc/lex parser with antlr parser
#	Content
1	#include <string>
2	#include <list>
3	#include <vector>
4	#include <map>
5	#include <utility>
6	#include <iostream>
7	#include <iterator>
8	#include <sstream>
9	#include <cassert>
10
11	#include <antlr/config.hpp>
12
13	#include <antlr/TokenStream.hpp>
14	#include <antlr/TokenWithIndex.hpp>
15	#include <antlr/BitSet.hpp>
16	#include <antlr/TokenStreamRewriteEngine.hpp>
17
18	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
19	namespace antlr {
20	#endif
21
22	const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";
23
24	namespace {
25
26	struct compareOperationIndex {
27	typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
28	bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
29	{
30	return a->getIndex() < b->getIndex();
31	}
32	};
33	struct dumpTokenWithIndex {
34	dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
35	void operator() ( const RefTokenWithIndex& t ) {
36	out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
37	}
38	ANTLR_USE_NAMESPACE(std)ostream& out;
39	};
40	};
41
42	TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
43	: stream(upstream)
44	, index(MIN_TOKEN_INDEX)
45	, tokens()
46	, programs()
47	, discardMask()
48	{
49	}
50
51	TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
52	: stream(upstream)
53	, index(MIN_TOKEN_INDEX)
54	, tokens(initialSize)
55	, programs()
56	, discardMask()
57	{
58	}
59
60	RefToken TokenStreamRewriteEngine::nextToken( void )
61	{
62	RefTokenWithIndex t;
63	// suck tokens until end of stream or we find a non-discarded token
64	do {
65	t = RefTokenWithIndex(stream.nextToken());
66	if ( t )
67	{
68	t->setIndex(index); // what is t's index in list?
69	if ( t->getType() != Token::EOF_TYPE ) {
70	tokens.push_back(t); // track all tokens except EOF
71	}
72	index++; // move to next position
73	}
74	} while ( t && discardMask.member(t->getType()) );
75	return RefToken(t);
76	}
77
78	void TokenStreamRewriteEngine::rollback( const std::string& programName,
79	size_t instructionIndex )
80	{
81	program_map::iterator rewrite = programs.find(programName);
82	if( rewrite != programs.end() )
83	{
84	operation_list& prog = rewrite->second;
85	operation_list::iterator
86	j = prog.begin(),
87	end = prog.end();
88
89	std::advance(j,instructionIndex);
90	if( j != end )
91	prog.erase(j, end);
92	}
93	}
94
95	void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
96	size_t start,
97	size_t end ) const
98	{
99	token_list::const_iterator s = tokens.begin();
100	std::advance( s, start );
101	token_list::const_iterator e = s;
102	std::advance( e, end-start );
103	std::for_each( s, e, tokenToStream(out) );
104	}
105
106	void TokenStreamRewriteEngine::toStream( std::ostream& out,
107	const std::string& programName,
108	size_t firstToken,
109	size_t lastToken ) const
110	{
111	if( tokens.size() == 0 )
112	return;
113
114	program_map::const_iterator rewriter = programs.find(programName);
115
116	if ( rewriter == programs.end() )
117	return;
118
119	// get the prog and some iterators in it...
120	const operation_list& prog = rewriter->second;
121	operation_list::const_iterator
122	rewriteOpIndex = prog.begin(),
123	rewriteOpEnd = prog.end();
124
125	size_t tokenCursor = firstToken;
126	// make sure we don't run out of the tokens we have...
127	if( lastToken > (tokens.size() - 1) )
128	lastToken = tokens.size() - 1;
129
130	while ( tokenCursor <= lastToken )
131	{
132	// std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;
133
134	if( rewriteOpIndex != rewriteOpEnd )
135	{
136	size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
137	while( tokenCursor < up_to_here )
138	out << tokens[tokenCursor++]->getText();
139	}
140	while ( rewriteOpIndex != rewriteOpEnd &&
141	tokenCursor == (*rewriteOpIndex)->getIndex() &&
142	tokenCursor <= lastToken )
143	{
144	tokenCursor = (*rewriteOpIndex)->execute(out);
145	++rewriteOpIndex;
146	}
147	if( tokenCursor <= lastToken )
148	out << tokens[tokenCursor++]->getText();
149	}
150	// std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
151	// now see if there are operations (append) beyond last token index
152	std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
153	rewriteOpIndex = rewriteOpEnd;
154	}
155
156	void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
157	size_t start,
158	size_t end ) const
159	{
160	token_list::const_iterator s = tokens.begin();
161	std::advance( s, start );
162	token_list::const_iterator e = s;
163	std::advance( e, end-start );
164	std::for_each( s, e, dumpTokenWithIndex(out) );
165	}
166
167	void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
168	RewriteOperation* op )
169	{
170	program_map::iterator rewrites = programs.find(programName);
171	// check if we got the program already..
172	if ( rewrites == programs.end() )
173	{
174	// no prog make a new one...
175	operation_list ops;
176	ops.push_back(op);
177	programs.insert(std::make_pair(programName,ops));
178	return;
179	}
180	operation_list& prog = rewrites->second;
181
182	if( prog.empty() )
183	{
184	prog.push_back(op);
185	return;
186	}
187
188	operation_list::iterator i, end = prog.end();
189	i = end;
190	--i;
191	// if at or beyond last op's index, just append
192	if ( op->getIndex() >= (*i)->getIndex() ) {
193	prog.push_back(op); // append to list of operations
194	return;
195	}
196	i = prog.begin();
197
198	if( i != end )
199	{
200	operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
201	prog.insert(pos,op);
202	}
203	else
204	prog.push_back(op);
205	}
206
207	#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
208	}
209	#endif