1ANTLR_BEGIN_NAMESPACE() 2 3template<class ImplTraits> 4TokenSource<ImplTraits>::TokenSource() 5 :m_eofToken( ImplTraits::CommonTokenType::TOKEN_EOF), 6 m_skipToken( ImplTraits::CommonTokenType::TOKEN_INVALID) 7{ 8} 9 10template<class ImplTraits> 11ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_eofToken() 12{ 13 return m_eofToken; 14} 15 16template<class ImplTraits> 17ANTLR_INLINE const typename TokenSource<ImplTraits>::TokenType& TokenSource<ImplTraits>::get_eofToken() const 18{ 19 return m_eofToken; 20} 21 22template<class ImplTraits> 23ANTLR_INLINE typename TokenSource<ImplTraits>::CommonTokenType& TokenSource<ImplTraits>::get_skipToken() 24{ 25 return m_skipToken; 26} 27 28template<class ImplTraits> 29ANTLR_INLINE typename TokenSource<ImplTraits>::StringType& TokenSource<ImplTraits>::get_fileName() 30{ 31 return m_fileName; 32} 33 34template<class ImplTraits> 35ANTLR_INLINE void TokenSource<ImplTraits>::set_fileName( const StringType& fileName ) 36{ 37 m_fileName = fileName; 38} 39 40template<class ImplTraits> 41typename TokenSource<ImplTraits>::LexerType* TokenSource<ImplTraits>::get_super() 42{ 43 return static_cast<LexerType*>(this); 44} 45 46template<class ImplTraits> 47typename TokenSource<ImplTraits>::TokenType* TokenSource<ImplTraits>::nextTokenStr() 48{ 49 typedef typename LexerType::RecognizerSharedStateType RecognizerSharedStateType; 50 typedef typename LexerType::InputStreamType InputStreamType; 51 typedef typename LexerType::IntStreamType IntStreamType; 52 LexerType* lexer; 53 RecognizerSharedStateType* state; 54 InputStreamType* input; 55 IntStreamType* istream; 56 57 lexer = this->get_super(); 58 state = lexer->get_rec()->get_state(); 59 input = lexer->get_input(); 60 istream = input->get_istream(); 61 62 /// Loop until we get a non skipped token or EOF 63 /// 64 for (;;) 65 { 66 // Get rid of any previous token (token factory takes care of 67 // any de-allocation when this token is finally used up. 68 // 69 state->set_token_present(false); 70 state->set_error(false); // Start out without an exception 71 state->set_failed(false); 72 73 // Now call the matching rules and see if we can generate a new token 74 // 75 for (;;) 76 { 77 // Record the start of the token in our input stream. 78 // 79 state->set_channel( TOKEN_DEFAULT_CHANNEL ); 80 state->set_tokenStartCharIndex( (ANTLR_MARKER)input->get_nextChar() ); 81 state->set_tokenStartCharPositionInLine( input->get_charPositionInLine() ); 82 state->set_tokenStartLine( input->get_line() ); 83 state->set_text(""); 84 85 if (istream->_LA(1) == ANTLR_CHARSTREAM_EOF) 86 { 87 // Reached the end of the current stream, nothing more to do if this is 88 // the last in the stack. 89 // 90 TokenType& teof = m_eofToken; 91 92 teof.set_startIndex(lexer->getCharIndex()); 93 teof.set_stopIndex(lexer->getCharIndex()); 94 teof.set_line(lexer->getLine()); 95 return &teof; 96 } 97 98 state->set_token_present( false ); 99 state->set_error(false); // Start out without an exception 100 state->set_failed(false); 101 102 // Call the generated lexer, see if it can get a new token together. 103 // 104 lexer->mTokens(); 105 106 if (state->get_error() == true) 107 { 108 // Recognition exception, report it and try to recover. 109 // 110 state->set_failed(true); 111 lexer->get_rec()->reportError(); 112 lexer->recover(); 113 } 114 else 115 { 116 if ( !state->get_token_present() ) 117 { 118 // Emit the real token, which adds it in to the token stream basically 119 // 120 lexer->emit(); 121 } 122 else if ( *(state->get_token()) == m_skipToken ) 123 { 124 // A real token could have been generated, but "Computer say's naaaaah" and it 125 // it is just something we need to skip altogether. 126 // 127 continue; 128 } 129 130 // Good token, not skipped, not EOF token 131 // 132 return state->get_token(); 133 } 134 } 135 } 136} 137 138template<class ImplTraits> 139typename TokenSource<ImplTraits>::TokenType* TokenSource<ImplTraits>::nextToken() 140{ 141 return this->nextToken( BoolForwarder<LexerType::IsFiltered>() ); 142} 143 144template<class ImplTraits> 145typename TokenSource<ImplTraits>::CommonTokenType* TokenSource<ImplTraits>::nextToken( BoolForwarder<true> /*isFiltered*/ ) 146{ 147 LexerType* lexer; 148 typename LexerType::RecognizerSharedStateType* state; 149 150 lexer = this->get_super(); 151 state = lexer->get_lexstate(); 152 153 /* Get rid of any previous token (token factory takes care of 154 * any deallocation when this token is finally used up. 155 */ 156 state->set_token_present( false ); 157 state->set_error( false ); /* Start out without an exception */ 158 state->set_failed(false); 159 160 /* Record the start of the token in our input stream. 161 */ 162 state->set_tokenStartCharIndex( lexer->index() ); 163 state->set_tokenStartCharPositionInLine( lexer->getCharPositionInLine() ); 164 state->set_tokenStartLine( lexer->getLine() ); 165 state->set_text(""); 166 167 /* Now call the matching rules and see if we can generate a new token 168 */ 169 for (;;) 170 { 171 if (lexer->LA(1) == ANTLR_CHARSTREAM_EOF) 172 { 173 /* Reached the end of the stream, nothing more to do. 174 */ 175 CommonTokenType& teof = m_eofToken; 176 177 teof.set_startIndex(lexer->getCharIndex()); 178 teof.set_stopIndex(lexer->getCharIndex()); 179 teof.set_line(lexer->getLine()); 180 return &teof; 181 } 182 183 state->set_token_present(false); 184 state->set_error(false); /* Start out without an exception */ 185 186 { 187 ANTLR_MARKER m; 188 189 m = lexer->get_istream()->mark(); 190 state->set_backtracking(1); /* No exceptions */ 191 state->set_failed(false); 192 193 /* Call the generated lexer, see if it can get a new token together. 194 */ 195 lexer->mTokens(); 196 state->set_backtracking(0); 197 198 /* mTokens backtracks with synpred at BACKTRACKING==2 199 and we set the synpredgate to allow actions at level 1. */ 200 201 if(state->get_failed()) 202 { 203 lexer->rewind(m); 204 lexer->consume(); //<! advance one char and try again !> 205 } 206 else 207 { 208 lexer->emit(); /* Assemble the token and emit it to the stream */ 209 TokenType* tok = state->get_token(); 210 return tok; 211 } 212 } 213 } 214} 215 216template<class ImplTraits> 217typename TokenSource<ImplTraits>::CommonTokenType* TokenSource<ImplTraits>::nextToken( BoolForwarder<false> /*isFiltered*/ ) 218{ 219 // Find the next token in the current stream 220 // 221 CommonTokenType* tok = this->nextTokenStr(); 222 223 // If we got to the EOF token then switch to the previous 224 // input stream if there were any and just return the 225 // EOF if there are none. We must check the next token 226 // in any outstanding input stream we pop into the active 227 // role to see if it was sitting at EOF after PUSHing the 228 // stream we just consumed, otherwise we will return EOF 229 // on the reinstalled input stream, when in actual fact 230 // there might be more input streams to POP before the 231 // real EOF of the whole logical inptu stream. Hence we 232 // use a while loop here until we find somethign in the stream 233 // that isn't EOF or we reach the actual end of the last input 234 // stream on the stack. 235 // 236 while(tok->get_type() == CommonTokenType::TOKEN_EOF) 237 { 238 typename ImplTraits::LexerType* lexer; 239 lexer = static_cast<typename ImplTraits::LexerType*>( this->get_super() ); 240 241 if ( lexer->get_rec()->get_state()->get_streams().size() > 0) 242 { 243 // We have another input stream in the stack so we 244 // need to revert to it, then resume the loop to check 245 // it wasn't sitting at EOF itself. 246 // 247 lexer->popCharStream(); 248 tok = this->nextTokenStr(); 249 } 250 else 251 { 252 // There were no more streams on the input stack 253 // so this EOF is the 'real' logical EOF for 254 // the input stream. So we just exit the loop and 255 // return the EOF we have found. 256 // 257 break; 258 } 259 260 } 261 262 // return whatever token we have, which may be EOF 263 // 264 return tok; 265} 266 267template<class ImplTraits> 268TokenStream<ImplTraits>::TokenStream() 269{ 270 m_tokenSource = NULL; 271 m_debugger = NULL; 272 m_initialStreamState = false; 273} 274 275template<class ImplTraits> 276typename TokenStream<ImplTraits>::IntStreamType* TokenStream<ImplTraits>::get_istream() 277{ 278 return this; 279} 280 281template<class ImplTraits> 282TokenStream<ImplTraits>::TokenStream(TokenSourceType* source, DebugEventListenerType* debugger) 283{ 284 m_initialStreamState = false; 285 m_tokenSource = source; 286 m_debugger = debugger; 287} 288 289template<class ImplTraits> 290CommonTokenStream<ImplTraits>::CommonTokenStream(ANTLR_UINT32 , TokenSourceType* source, 291 DebugEventListenerType* debugger) 292 : CommonTokenStream<ImplTraits>::BaseType( source, debugger ) 293{ 294 m_p = -1; 295 m_channel = TOKEN_DEFAULT_CHANNEL; 296 m_discardOffChannel = false; 297 m_nissued = 0; 298} 299 300template<class ImplTraits> 301typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens() 302{ 303 return m_tokens; 304} 305 306template<class ImplTraits> 307const typename CommonTokenStream<ImplTraits>::TokensType& CommonTokenStream<ImplTraits>::get_tokens() const 308{ 309 return m_tokens; 310} 311 312template<class ImplTraits> 313typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet() 314{ 315 return m_discardSet; 316} 317 318template<class ImplTraits> 319const typename CommonTokenStream<ImplTraits>::DiscardSetType& CommonTokenStream<ImplTraits>::get_discardSet() const 320{ 321 return m_discardSet; 322} 323 324template<class ImplTraits> 325ANTLR_INLINE ANTLR_INT32 CommonTokenStream<ImplTraits>::get_p() const 326{ 327 return m_p; 328} 329 330template<class ImplTraits> 331ANTLR_INLINE void CommonTokenStream<ImplTraits>::set_p( ANTLR_INT32 p ) 332{ 333 m_p = p; 334} 335 336template<class ImplTraits> 337ANTLR_INLINE void CommonTokenStream<ImplTraits>::inc_p() 338{ 339 ++m_p; 340} 341 342template<class ImplTraits> 343ANTLR_INLINE void CommonTokenStream<ImplTraits>::dec_p() 344{ 345 --m_p; 346} 347 348template<class ImplTraits> 349ANTLR_INLINE ANTLR_MARKER CommonTokenStream<ImplTraits>::index_impl() 350{ 351 return m_p; 352} 353 354// Reset a token stream so it can be used again and can reuse it's 355// resources. 356// 357template<class ImplTraits> 358void CommonTokenStream<ImplTraits>::reset() 359{ 360 // Free any resources that ar most like specifc to the 361 // run we just did. 362 // 363 m_discardSet.clear(); 364 m_channelOverrides.clear(); 365 366 // Now, if there were any existing tokens in the stream, 367 // then we just reset the vector count so that it starts 368 // again. We must traverse the entries unfortunately as 369 // there may be free pointers for custom token types and 370 // so on. However that is just a quick NULL check on the 371 // vector entries. 372 // 373 m_tokens.clear(); 374 375 // Reset to defaults 376 // 377 m_discardOffChannel = false; 378 m_channel = ImplTraits::CommonTokenType::TOKEN_DEFAULT_CHANNEL; 379 m_p = -1; 380} 381 382template<class ImplTraits> 383void TokenStream<ImplTraits>::setDebugListener(DebugEventListenerType* debugger) 384{ 385 m_debugger = debugger; 386 m_initialStreamState = false; 387} 388 389template<class ImplTraits> 390const typename TokenStream<ImplTraits>::TokenType* TokenStream<ImplTraits>::_LT(ANTLR_INT32 k) 391{ 392 ANTLR_INT32 i; 393 ANTLR_INT32 n; 394 TokenStreamType* cts; 395 396 cts = this->get_super(); 397 398 if(k < 0) 399 { 400 return cts->LB(-k); 401 } 402 403 ANTLR_INT32 req_idx = cts->get_p() + k - 1; 404 ANTLR_INT32 cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize()); 405 406 if( (cts->get_p() == -1) || 407 ( ( req_idx >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) ) 408 ) 409 { 410 cts->fillBuffer(); 411 } 412 413 // Here we used to check for k == 0 and return 0, but this seems 414 // a superfluous check to me. LT(k=0) is therefore just undefined 415 // and we won't waste the clock cycles on the check 416 // 417 cached_size = static_cast<ANTLR_INT32>(this->get_istream()->get_cachedSize()); 418 if ( req_idx >= cached_size ) 419 { 420 TokenType& teof = cts->get_tokenSource()->get_eofToken(); 421 422 teof.set_startIndex( this->get_istream()->index()); 423 teof.set_stopIndex( this->get_istream()->index()); 424 return &teof; 425 } 426 427 i = cts->get_p(); 428 n = 1; 429 430 /* Need to find k good tokens, skipping ones that are off channel 431 */ 432 while( n < k) 433 { 434 /* Skip off-channel tokens */ 435 i = cts->skipOffTokenChannels(i+1); /* leave p on valid token */ 436 n++; 437 } 438 439 if( ( i >= cached_size ) && ( (cached_size % ImplTraits::TOKEN_FILL_BUFFER_INCREMENT) == 0 ) ) 440 { 441 cts->fillBuffer(); 442 } 443 if ( (ANTLR_UINT32) i >= this->get_istream()->get_cachedSize() ) 444 { 445 TokenType& teof = cts->get_tokenSource()->get_eofToken(); 446 447 teof.set_startIndex(this->get_istream()->index()); 448 teof.set_stopIndex(this->get_istream()->index()); 449 return &teof; 450 } 451 452 // Here the token must be in the input vector. Rather then incur 453 // function call penalty, we just return the pointer directly 454 // from the vector 455 // 456 return cts->getToken(i); 457} 458 459template<class ImplTraits> 460const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::LB(ANTLR_INT32 k) 461{ 462 ANTLR_INT32 i; 463 ANTLR_INT32 n; 464 465 if (m_p == -1) 466 { 467 this->fillBuffer(); 468 } 469 if (k == 0) 470 { 471 return NULL; 472 } 473 if ((m_p - k) < 0) 474 { 475 return NULL; 476 } 477 478 i = m_p; 479 n = 1; 480 481 /* Need to find k good tokens, going backwards, skipping ones that are off channel 482 */ 483 while (n <= k) 484 { 485 /* Skip off-channel tokens 486 */ 487 488 i = this->skipOffTokenChannelsReverse(i - 1); /* leave p on valid token */ 489 n++; 490 } 491 if (i < 0) 492 { 493 return NULL; 494 } 495 496 // Here the token must be in the input vector. Rather then incut 497 // function call penalty, we jsut return the pointer directly 498 // from the vector 499 // 500 return this->getToken(i); 501} 502 503template<class ImplTraits> 504const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken(ANTLR_MARKER i) 505{ 506 return this->get(i); 507} 508 509 510template<class ImplTraits> 511const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::get(ANTLR_MARKER i) 512{ 513 return this->getToken( static_cast<ANTLR_MARKER>(i), 514 BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>() ); 515} 516 517template<class ImplTraits> 518const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx, 519 BoolForwarder<true> /*tokens_accessed_from_owning_rule*/ ) 520{ 521 typename TokensType::iterator iter = m_tokens.find(tok_idx); 522 if( iter == m_tokens.end() ) 523 { 524 TokenAccessException ex; 525 throw ex; 526 } 527 const TokenType& tok = iter->second; 528 return &tok; 529} 530 531template<class ImplTraits> 532const typename CommonTokenStream<ImplTraits>::TokenType* CommonTokenStream<ImplTraits>::getToken( ANTLR_MARKER tok_idx, BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ ) 533{ 534 TokenType& tok = m_tokens.at( static_cast<ANTLR_UINT32>(tok_idx) ); 535 return &tok; 536} 537 538template<class ImplTraits> 539typename TokenStream<ImplTraits>::TokenSourceType* TokenStream<ImplTraits>::get_tokenSource() const 540{ 541 return m_tokenSource; 542} 543 544template<class ImplTraits> 545void TokenStream<ImplTraits>::set_tokenSource( TokenSourceType* tokenSource ) 546{ 547 m_tokenSource = tokenSource; 548} 549 550template<class ImplTraits> 551typename TokenStream<ImplTraits>::StringType TokenStream<ImplTraits>::toString() 552{ 553 TokenStreamType* cts = static_cast<TokenStreamType>(this); 554 555 if (cts->get_p() == -1) 556 { 557 cts->fillBuffer(); 558 } 559 560 return this->toStringSS(0, this->get_istream()->size()); 561} 562 563template<class ImplTraits> 564typename TokenStream<ImplTraits>::StringType 565TokenStream<ImplTraits>::toStringSS(ANTLR_MARKER start, ANTLR_MARKER stop) 566{ 567 StringType string; 568 TokenSourceType* tsource; 569 const TokenType* tok; 570 TokenStreamType* cts; 571 572 cts = this->get_super(); 573 574 if (cts->get_p() == -1) 575 { 576 cts->fillBuffer(); 577 } 578 if (stop >= this->get_istream()->size()) 579 { 580 stop = this->get_istream()->size() - 1; 581 } 582 583 /* Who is giving us these tokens? 584 */ 585 tsource = cts->get_tokenSource(); 586 587 if (tsource != NULL && !cts->get_tokens().empty() ) 588 { 589 /* Finally, let's get a string 590 */ 591 for (ANTLR_MARKER i = start; i <= stop; i++) 592 { 593 tok = cts->get(i); 594 if (tok != NULL) 595 { 596 string.append( tok->getText() ); 597 } 598 } 599 600 return string; 601 } 602 return ""; 603} 604 605template<class ImplTraits> 606typename TokenStream<ImplTraits>::StringType 607TokenStream<ImplTraits>::toStringTT(const TokenType* start, const TokenType* stop) 608{ 609 if (start != NULL && stop != NULL) 610 { 611 return this->toStringSS( start->get_tokenIndex(), 612 stop->get_tokenIndex()); 613 } 614 else 615 { 616 return ""; 617 } 618} 619 620/** A simple filter mechanism whereby you can tell this token stream 621 * to force all tokens of type ttype to be on channel. For example, 622 * when interpreting, we cannot execute actions so we need to tell 623 * the stream to force all WS and NEWLINE to be a different, ignored, 624 * channel. 625 */ 626template<class ImplTraits> 627void CommonTokenStream<ImplTraits>::setTokenTypeChannel ( ANTLR_UINT32 ttype, ANTLR_UINT32 channel) 628{ 629 /* We add one to the channel so we can distinguish NULL as being no entry in the 630 * table for a particular token type. 631 */ 632 m_channelOverrides[ttype] = (ANTLR_UINT32)channel + 1; 633 634} 635 636template<class ImplTraits> 637void CommonTokenStream<ImplTraits>::discardTokenType(ANTLR_INT32 ttype) 638{ 639 /* We add one to the channel so we can distinguish NULL as being no entry in the 640 * table for a particular token type. We could use bitsets for this I suppose too. 641 */ 642 m_discardSet.insert(ttype); 643} 644 645template<class ImplTraits> 646void CommonTokenStream<ImplTraits>::discardOffChannelToks(bool discard) 647{ 648 m_discardOffChannel = discard; 649} 650 651template<class ImplTraits> 652typename CommonTokenStream<ImplTraits>::TokensType* CommonTokenStream<ImplTraits>::getTokens() 653{ 654 if (m_p == -1) 655 { 656 this->fillBuffer(); 657 } 658 659 return &m_tokens; 660} 661 662template<class ImplTraits> 663void CommonTokenStream<ImplTraits>::getTokenRange(ANTLR_UINT32 start, ANTLR_UINT32 stop, 664 TokensListType& tokenRange) 665{ 666 return this->getTokensSet(start, stop, NULL, tokenRange); 667} 668 669/** Given a start and stop index, return a List of all tokens in 670 * the token type BitSet. Return null if no tokens were found. This 671 * method looks at both on and off channel tokens. 672 */ 673template<class ImplTraits> 674void 675CommonTokenStream<ImplTraits>::getTokensSet(ANTLR_UINT32 start, ANTLR_UINT32 stop, BitsetType* types, 676 TokensListType& filteredList ) 677{ 678 ANTLR_UINT32 i; 679 ANTLR_UINT32 n; 680 TokenType* tok; 681 682 if ( m_p == -1) 683 { 684 this->fillBuffer(); 685 } 686 if (stop > this->get_istream()->size()) 687 { 688 stop = this->get_istream()->size(); 689 } 690 if (start > stop) 691 { 692 return; 693 } 694 695 /* We have the range set, now we need to iterate through the 696 * installed tokens and create a new list with just the ones we want 697 * in it. We are just moving pointers about really. 698 */ 699 for(i = start, n = 0; i<= stop; i++) 700 { 701 tok = this->get(i); 702 703 if ( types == NULL 704 || (types->isMember( tok->get_type() ) == true ) 705 ) 706 { 707 filteredList.push_back(tok); 708 } 709 } 710 711 return ; 712} 713 714template<class ImplTraits> 715void 716CommonTokenStream<ImplTraits>::getTokensList(ANTLR_UINT32 start, ANTLR_UINT32 stop, 717 const IntListType& list, TokensListType& newlist) 718{ 719 BitsetType* bitSet; 720 721 bitSet = Bitset<ImplTraits>::BitsetFromList(list); 722 this->getTokensSet(start, stop, bitSet, newlist); 723 delete bitSet; 724} 725 726template<class ImplTraits> 727void 728CommonTokenStream<ImplTraits>::getTokensType(ANTLR_UINT32 start, ANTLR_UINT32 stop, ANTLR_UINT32 type, 729 TokensListType& newlist ) 730{ 731 BitsetType* bitSet; 732 733 bitSet = BitsetType::BitsetOf(type, -1); 734 this->getTokensSet(start, stop, bitSet, newlist); 735 736 delete bitSet; 737} 738 739template<class ImplTraits> 740void CommonTokenStream<ImplTraits>::fillBufferExt() 741{ 742 this->fillBuffer(); 743} 744 745template<class ImplTraits> 746bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32 cnt, 747 BoolForwarder<true> ) 748{ 749 return ( cnt >= ImplTraits::TOKEN_FILL_BUFFER_INCREMENT ); 750} 751 752template<class ImplTraits> 753bool CommonTokenStream<ImplTraits>::hasReachedFillbufferTarget( ANTLR_UINT32, 754 BoolForwarder<false> ) 755{ 756 return false; 757} 758 759 760template<class ImplTraits> 761void CommonTokenStream<ImplTraits>::fillBuffer() 762{ 763 ANTLR_UINT32 index; 764 TokenType* tok; 765 bool discard; 766 767 /* Start at index 0 of course 768 */ 769 ANTLR_UINT32 cached_p = (m_p < 0) ? 0 : m_p; 770 index = m_nissued; 771 ANTLR_UINT32 cnt = 0; 772 773 /* Pick out the next token from the token source 774 * Remember we just get a pointer (reference if you like) here 775 * and so if we store it anywhere, we don't set any pointers to auto free it. 776 */ 777 tok = this->get_tokenSource()->nextToken(); 778 779 while ( tok->get_type() != TokenType::TOKEN_EOF ) 780 { 781 discard = false; /* Assume we are not discarding */ 782 783 /* I employ a bit of a trick, or perhaps hack here. Rather than 784 * store a pointer to a structure in the override map and discard set 785 * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 786 * we can distinguish "not being there" from "being channel or type 0" 787 */ 788 789 if ( m_discardSet.find(tok->get_type()) != m_discardSet.end() ) 790 { 791 discard = true; 792 } 793 else if ( m_discardOffChannel == true 794 && tok->get_channel() != m_channel 795 ) 796 { 797 discard = true; 798 } 799 else if (!m_channelOverrides.empty()) 800 { 801 /* See if this type is in the override map 802 */ 803 typename ChannelOverridesType::iterator iter = m_channelOverrides.find( tok->get_type() + 1 ); 804 805 if (iter != m_channelOverrides.end()) 806 { 807 /* Override found 808 */ 809 tok->set_channel( ANTLR_UINT32_CAST(iter->second) - 1); 810 } 811 } 812 813 /* If not discarding it, add it to the list at the current index 814 */ 815 if (discard == false) 816 { 817 /* Add it, indicating that we will delete it and the table should not 818 */ 819 tok->set_tokenIndex(index); 820 ++m_p; 821 this->insertToken(*tok); 822 index++; 823 m_nissued++; 824 cnt++; 825 } 826 827 if( !this->hasReachedFillbufferTarget( cnt, 828 BoolForwarder<ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE>() ) ) 829 tok = this->get_tokenSource()->nextToken(); 830 else 831 break; 832 } 833 834 /* Cache the size so we don't keep doing indirect method calls. We do this as 835 * early as possible so that anything after this may utilize the cached value. 836 */ 837 this->get_istream()->set_cachedSize( m_nissued ); 838 839 /* Set the consume pointer to the first token that is on our channel, we just read 840 */ 841 m_p = cached_p; 842 m_p = this->skipOffTokenChannels( m_p ); 843 844} 845/// Given a starting index, return the index of the first on-channel 846/// token. 847/// 848template<class ImplTraits> 849ANTLR_UINT32 CommonTokenStream<ImplTraits>::skipOffTokenChannels(ANTLR_INT32 i) 850{ 851 ANTLR_INT32 n; 852 n = this->get_istream()->get_cachedSize(); 853 854 while (i < n) 855 { 856 const TokenType* tok = this->getToken(i); 857 858 if (tok->get_channel() != m_channel ) 859 { 860 i++; 861 } 862 else 863 { 864 return i; 865 } 866 } 867 return i; 868} 869 870template<class ImplTraits> 871ANTLR_UINT32 CommonTokenStream<ImplTraits>::skipOffTokenChannelsReverse(ANTLR_INT32 x) 872{ 873 while (x >= 0) 874 { 875 const TokenType* tok = this->getToken(x); 876 877 if( tok->get_channel() != m_channel ) 878 { 879 x--; 880 } 881 else 882 { 883 return x; 884 } 885 } 886 return x; 887} 888 889template<class ImplTraits> 890void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop ) 891{ 892 this->discardTokens( start, stop, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() ); 893} 894 895template<class ImplTraits> 896void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop, 897 BoolForwarder<true> /*tokens_accessed_from_owning_rule */ ) 898{ 899 typename TokensType::iterator iter1 = m_tokens.lower_bound(start); 900 typename TokensType::iterator iter2 = m_tokens.upper_bound(stop); 901 m_tokens.erase( iter1, iter2 ); 902} 903 904template<class ImplTraits> 905void CommonTokenStream<ImplTraits>::discardTokens( ANTLR_MARKER start, ANTLR_MARKER stop, 906 BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ ) 907{ 908 m_tokens.erase( m_tokens.begin() + start, m_tokens.begin() + stop ); 909} 910 911template<class ImplTraits> 912void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok ) 913{ 914 this->insertToken( tok, BoolForwarder< ImplTraits::TOKENS_ACCESSED_FROM_OWNING_RULE >() ); 915} 916 917template<class ImplTraits> 918void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<true> /*tokens_accessed_from_owning_rule*/ ) 919{ 920 assert( m_tokens.find( tok.get_index() ) == m_tokens.end() ); 921 assert( tok.get_index() == m_nissued ); 922 m_tokens[ tok.get_index() ] = tok; 923} 924 925template<class ImplTraits> 926void CommonTokenStream<ImplTraits>::insertToken( const TokenType& tok, BoolForwarder<false> /*tokens_accessed_from_owning_rule*/ ) 927{ 928 m_tokens.push_back( tok ); 929} 930 931template<class ImplTraits> 932CommonTokenStream<ImplTraits>::~CommonTokenStream() 933{ 934 m_tokens.clear(); 935} 936 937ANTLR_END_NAMESPACE() 938