1ANTLR_BEGIN_NAMESPACE() 2 3template<class ImplTraits> 4Lexer<ImplTraits>::Lexer(ANTLR_UINT32 sizeHint, RecognizerSharedStateType* state) 5 :Lexer<ImplTraits>::RecognizerType(sizeHint, state) 6 ,m_input(NULL) 7{ 8} 9 10template<class ImplTraits> 11Lexer<ImplTraits>::Lexer(ANTLR_UINT32 sizeHint, InputStreamType* input, RecognizerSharedStateType* state) 12 :Lexer<ImplTraits>::RecognizerType(sizeHint, state) 13{ 14 this->setCharStream(input); 15} 16 17template<class ImplTraits> 18typename Lexer<ImplTraits>::InputStreamType* Lexer<ImplTraits>::get_input() const 19{ 20 return m_input; 21} 22 23template<class ImplTraits> 24typename Lexer<ImplTraits>::IntStreamType* Lexer<ImplTraits>::get_istream() const 25{ 26 return m_input; 27} 28 29template<class ImplTraits> 30typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_rec() 31{ 32 return this; 33} 34 35template<class ImplTraits> 36typename Lexer<ImplTraits>::TokenSourceType* Lexer<ImplTraits>::get_tokSource() 37{ 38 return this; 39} 40 41template<class ImplTraits> 42void Lexer<ImplTraits>::displayRecognitionError( ANTLR_UINT8** , ExceptionBaseType* ex) 43{ 44 StringStreamType err_stream; 45 46 // See if there is a 'filename' we can use 47 // 48 if( ex->getName().empty() ) 49 { 50 err_stream << "-unknown source-("; 51 } 52 else 53 { 54 err_stream << ex->get_streamName().c_str(); 55 err_stream << "("; 56 } 57 err_stream << ex->get_line() << ")"; 58 59 err_stream << ": lexer error " << ex->getName() << '(' << ex->getType() << ')' << " :\n\t" 60 << ex->get_message() << " at position [" << ex->get_line() << ", " 61 << ex->get_charPositionInLine()+1 << "], "; 62 63 { 64 ANTLR_UINT32 width; 65 66 width = ANTLR_UINT32_CAST(( (ANTLR_UINT8*)(m_input->get_data()) + 67 (m_input->size() )) - (ANTLR_UINT8*)( ex->get_index() )); 68 69 if (width >= 1) 70 { 71 if (isprint(ex->get_c() )) 72 { 73 err_stream << "near '" << (typename StringType::value_type) ex->get_c() << "' :\n"; 74 } 75 else 76 { 77 err_stream << "near char(" << std::hex << ex->get_c() << std::dec << ") :\n"; 78 } 79 err_stream << "\t"; 80 err_stream.width( width > 20 ? 20 : width ); 81 err_stream << (typename StringType::const_pointer)ex->get_index() << "\n"; 82 } 83 else 84 { 85 err_stream << "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n"; 86 err_stream << "\t The lexer was matching from line " 87 << this->get_state()->get_tokenStartLine() 88 << ", offset " << this->get_state()->get_tokenStartCharPositionInLine() 89 << ", which\n\t "; 90 width = ANTLR_UINT32_CAST(((ANTLR_UINT8*)(m_input->get_data() )+ 91 (m_input->size())) - 92 (ANTLR_UINT8*)(this->get_state()->get_tokenStartCharIndex() )); 93 94 if (width >= 1) 95 { 96 err_stream << "looks like this:\n\t\t"; 97 err_stream.width( width > 20 ? 20 : width ); 98 err_stream << (typename StringType::const_pointer)this->get_state()->get_tokenStartCharIndex() << "\n"; 99 } 100 else 101 { 102 err_stream << "is also the end of the line, so you must check your lexer rules\n"; 103 } 104 } 105 } 106 ImplTraits::displayRecognitionError( err_stream.str() ); 107} 108 109template<class ImplTraits> 110void Lexer<ImplTraits>::fillExceptionData( ExceptionBaseType* ex ) 111{ 112 ex->set_c( m_input->_LA(1) ); /* Current input character */ 113 ex->set_line( m_input->get_line() ); /* Line number comes from stream */ 114 ex->set_charPositionInLine( m_input->get_charPositionInLine() ); /* Line offset also comes from the stream */ 115 ex->set_index( m_input->index() ); 116 ex->set_streamName( m_input->get_fileName() ); 117 ex->set_message( "Unexpected character" ); 118} 119 120template<class ImplTraits> 121void Lexer<ImplTraits>::setCharStream(InputStreamType* input) 122{ 123 /* Install the input interface 124 */ 125 m_input = input; 126 127 /* Set the current token to nothing 128 */ 129 RecognizerSharedStateType* state = this->get_rec()->get_state(); 130 state->set_token_present( false ); 131 state->set_text(""); 132 state->set_tokenStartCharIndex(-1); 133 134 /* Copy the name of the char stream to the token source 135 */ 136 this->get_tokSource()->set_fileName( input->get_fileName() ); 137} 138 139template<class ImplTraits> 140void Lexer<ImplTraits>::pushCharStream(InputStreamType* input) 141{ 142 // We have a stack, so we can save the current input stream 143 // into it. 144 // 145 this->get_istream()->mark(); 146 this->get_rec()->get_state()->get_streams().push(this->get_input()); 147 148 // And now we can install this new one 149 // 150 this->setCharStream(input); 151} 152 153template<class ImplTraits> 154void Lexer<ImplTraits>::popCharStream() 155{ 156 InputStreamType* input; 157 158 // If we do not have a stream stack or we are already at the 159 // stack bottom, then do nothing. 160 // 161 typename RecognizerSharedStateType::StreamsType& streams = this->get_rec()->get_state()->get_streams(); 162 if ( streams.size() > 0) 163 { 164 // We just leave the current stream to its fate, we do not close 165 // it or anything as we do not know what the programmer intended 166 // for it. This method can always be overridden of course. 167 // So just find out what was currently saved on the stack and use 168 // that now, then pop it from the stack. 169 // 170 input = streams.top(); 171 streams.pop(); 172 173 // Now install the stream as the current one. 174 // 175 this->setCharStream(input); 176 this->get_istream()->rewindLast(); 177 } 178 return; 179} 180 181template<class ImplTraits> 182void Lexer<ImplTraits>::emit(const CommonTokenType* token) 183{ 184 this->get_rec()->get_state()->set_token(token); 185} 186 187template<class ImplTraits> 188typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::emit() 189{ 190 /* We could check pointers to token factories and so on, but 191 * we are in code that we want to run as fast as possible 192 * so we are not checking any errors. So make sure you have installed an input stream before 193 * trying to emit a new token. 194 */ 195 RecognizerSharedStateType* state = this->get_rec()->get_state(); 196 state->set_token_present(true); 197 CommonTokenType* token = state->get_token(); 198 token->set_input( this->get_input() ); 199 200 /* Install the supplied information, and some other bits we already know 201 * get added automatically, such as the input stream it is associated with 202 * (though it can all be overridden of course) 203 */ 204 token->set_type( state->get_type() ); 205 token->set_channel( state->get_channel() ); 206 token->set_startIndex( state->get_tokenStartCharIndex() ); 207 token->set_stopIndex( this->getCharIndex() - 1 ); 208 token->set_line( state->get_tokenStartLine() ); 209 token->set_charPositionInLine( state->get_tokenStartCharPositionInLine() ); 210 211 token->set_tokText( state->get_text() ); 212 token->set_lineStart( this->get_input()->get_currentLine() ); 213 214 return token; 215} 216 217template<class ImplTraits> 218Lexer<ImplTraits>::~Lexer() 219{ 220 // This may have ben a delegate or delegator lexer, in which case the 221 // state may already have been freed (and set to NULL therefore) 222 // so we ignore the state if we don't have it. 223 // 224 RecognizerSharedStateType* state = this->get_rec()->get_state(); 225 226 if ( state != NULL) 227 { 228 state->get_streams().clear(); 229 } 230} 231 232template<class ImplTraits> 233bool Lexer<ImplTraits>::matchs(ANTLR_UCHAR* str ) 234{ 235 RecognizerSharedStateType* state = this->get_rec()->get_state(); 236 while (*str != ANTLR_STRING_TERMINATOR) 237 { 238 if ( this->get_istream()->_LA(1) != (*str)) 239 { 240 if ( state->get_backtracking() > 0) 241 { 242 state->set_failed(true); 243 return false; 244 } 245 246 this->exConstruct(); 247 state->set_failed( true ); 248 249 /* TODO: Implement exception creation more fully perhaps 250 */ 251 this->recover(); 252 return false; 253 } 254 255 /* Matched correctly, do consume it 256 */ 257 this->get_istream()->consume(); 258 str++; 259 260 } 261 /* Reset any failed indicator 262 */ 263 state->set_failed( false ); 264 return true; 265} 266 267template<class ImplTraits> 268bool Lexer<ImplTraits>::matchc(ANTLR_UCHAR c) 269{ 270 if (this->get_istream()->_LA(1) == c) 271 { 272 /* Matched correctly, do consume it 273 */ 274 this->get_istream()->consume(); 275 276 /* Reset any failed indicator 277 */ 278 this->get_rec()->get_state()->set_failed( false ); 279 280 return true; 281 } 282 283 /* Failed to match, exception and recovery time. 284 */ 285 if(this->get_rec()->get_state()->get_backtracking() > 0) 286 { 287 this->get_rec()->get_state()->set_failed( true ); 288 return false; 289 } 290 291 this->exConstruct(); 292 293 /* TODO: Implement exception creation more fully perhaps 294 */ 295 this->recover(); 296 297 return false; 298} 299 300template<class ImplTraits> 301bool Lexer<ImplTraits>::matchRange(ANTLR_UCHAR low, ANTLR_UCHAR high) 302{ 303 ANTLR_UCHAR c; 304 305 /* What is in the stream at the moment? 306 */ 307 c = this->get_istream()->_LA(1); 308 if ( c >= low && c <= high) 309 { 310 /* Matched correctly, consume it 311 */ 312 this->get_istream()->consume(); 313 314 /* Reset any failed indicator 315 */ 316 this->get_rec()->get_state()->set_failed( false ); 317 318 return true; 319 } 320 321 /* Failed to match, execption and recovery time. 322 */ 323 324 if (this->get_rec()->get_state()->get_backtracking() > 0) 325 { 326 this->get_rec()->get_state()->set_failed( true ); 327 return false; 328 } 329 330 this->exConstruct(); 331 332 /* TODO: Implement exception creation more fully 333 */ 334 this->recover(); 335 336 return false; 337} 338 339template<class ImplTraits> 340void Lexer<ImplTraits>::matchAny() 341{ 342 this->get_istream()->consume(); 343} 344 345template<class ImplTraits> 346void Lexer<ImplTraits>::recover() 347{ 348 this->get_istream()->consume(); 349} 350 351template<class ImplTraits> 352ANTLR_UINT32 Lexer<ImplTraits>::getLine() 353{ 354 return this->get_input()->get_line(); 355} 356 357template<class ImplTraits> 358ANTLR_MARKER Lexer<ImplTraits>::getCharIndex() 359{ 360 return this->get_istream()->index(); 361} 362 363template<class ImplTraits> 364ANTLR_UINT32 Lexer<ImplTraits>::getCharPositionInLine() 365{ 366 return this->get_input()->get_charPositionInLine(); 367} 368 369template<class ImplTraits> 370typename Lexer<ImplTraits>::StringType Lexer<ImplTraits>::getText() 371{ 372 RecognizerSharedStateType* state = this->get_rec()->get_state(); 373 if ( !state->get_text().empty() ) 374 { 375 return state->get_text(); 376 377 } 378 return this->get_input()->substr( state->get_tokenStartCharIndex(), 379 this->getCharIndex() - this->get_input()->get_charByteSize() 380 ); 381} 382 383template<class ImplTraits> 384void Lexer<ImplTraits>::exConstruct() 385{ 386 new ANTLR_Exception<ImplTraits, RECOGNITION_EXCEPTION, InputStreamType>( this->get_rec(), "" ); 387} 388 389template< class ImplTraits> 390typename Lexer<ImplTraits>::TokenType* Lexer<ImplTraits>::getMissingSymbol( IntStreamType*, 391 ExceptionBaseType*, 392 ANTLR_UINT32 , BitsetListType*) 393{ 394 return NULL; 395} 396 397template< class ImplTraits> 398ANTLR_INLINE const typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_rec() const 399{ 400 return this; 401} 402 403template< class ImplTraits> 404ANTLR_INLINE const typename Lexer<ImplTraits>::RecognizerType* Lexer<ImplTraits>::get_recognizer() const 405{ 406 return this->get_rec(); 407} 408 409template< class ImplTraits> 410ANTLR_INLINE typename Lexer<ImplTraits>::RecognizerSharedStateType* Lexer<ImplTraits>::get_lexstate() const 411{ 412 return this->get_rec()->get_state(); 413} 414 415template< class ImplTraits> 416ANTLR_INLINE void Lexer<ImplTraits>::set_lexstate( RecognizerSharedStateType* lexstate ) 417{ 418 this->get_rec()->set_state(lexstate); 419} 420 421template< class ImplTraits> 422ANTLR_INLINE const typename Lexer<ImplTraits>::TokenSourceType* Lexer<ImplTraits>::get_tokSource() const 423{ 424 return this; 425} 426 427template< class ImplTraits> 428ANTLR_INLINE typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::get_ltoken() const 429{ 430 return this->get_lexstate()->token(); 431} 432 433template< class ImplTraits> 434ANTLR_INLINE void Lexer<ImplTraits>::set_ltoken( const CommonTokenType* ltoken ) 435{ 436 this->get_lexstate()->set_token( ltoken ); 437} 438 439template< class ImplTraits> 440ANTLR_INLINE bool Lexer<ImplTraits>::hasFailed() const 441{ 442 return this->get_lexstate()->get_failed(); 443} 444 445template< class ImplTraits> 446ANTLR_INLINE ANTLR_INT32 Lexer<ImplTraits>::get_backtracking() const 447{ 448 return this->get_lexstate()->get_backtracking(); 449} 450 451template< class ImplTraits> 452ANTLR_INLINE void Lexer<ImplTraits>::inc_backtracking() 453{ 454 this->get_lexstate()->inc_backtracking(); 455} 456 457template< class ImplTraits> 458ANTLR_INLINE void Lexer<ImplTraits>::dec_backtracking() 459{ 460 this->get_lexstate()->dec_backtracking(); 461} 462 463template< class ImplTraits> 464ANTLR_INLINE bool Lexer<ImplTraits>::get_failedflag() const 465{ 466 return this->get_lexstate()->get_failed(); 467} 468 469template< class ImplTraits> 470ANTLR_INLINE void Lexer<ImplTraits>::set_failedflag( bool failed ) 471{ 472 this->get_lexstate()->set_failed(failed); 473} 474 475template< class ImplTraits> 476ANTLR_INLINE typename Lexer<ImplTraits>::InputStreamType* Lexer<ImplTraits>::get_strstream() const 477{ 478 return this->get_input(); 479} 480 481template< class ImplTraits> 482ANTLR_INLINE ANTLR_MARKER Lexer<ImplTraits>::index() const 483{ 484 return this->get_istream()->index(); 485} 486 487template< class ImplTraits> 488ANTLR_INLINE void Lexer<ImplTraits>::seek(ANTLR_MARKER index) 489{ 490 this->get_istream()->seek(index); 491} 492 493template< class ImplTraits> 494ANTLR_INLINE const typename Lexer<ImplTraits>::CommonTokenType* Lexer<ImplTraits>::EOF_Token() const 495{ 496 const CommonTokenType& eof_token = this->get_tokSource()->get_eofToken(); 497 return &eof_token; 498} 499 500template< class ImplTraits> 501ANTLR_INLINE bool Lexer<ImplTraits>::hasException() const 502{ 503 return this->get_lexstate()->get_error(); 504} 505 506template< class ImplTraits> 507ANTLR_INLINE typename Lexer<ImplTraits>::ExceptionBaseType* Lexer<ImplTraits>::get_exception() const 508{ 509 return this->get_lexstate()->get_exception(); 510} 511 512template< class ImplTraits> 513ANTLR_INLINE void Lexer<ImplTraits>::constructEx() 514{ 515 this->get_rec()->exConstruct(); 516} 517 518template< class ImplTraits> 519ANTLR_INLINE ANTLR_MARKER Lexer<ImplTraits>::mark() 520{ 521 return this->get_istream()->mark(); 522} 523 524template< class ImplTraits> 525ANTLR_INLINE void Lexer<ImplTraits>::rewind(ANTLR_MARKER marker) 526{ 527 this->get_istream()->rewind(marker); 528} 529 530template< class ImplTraits> 531ANTLR_INLINE void Lexer<ImplTraits>::rewindLast() 532{ 533 this->get_istream()->rewindLast(); 534} 535 536template< class ImplTraits> 537ANTLR_INLINE void Lexer<ImplTraits>::memoize(ANTLR_MARKER ruleIndex, ANTLR_MARKER ruleParseStart) 538{ 539 this->get_rec()->memoize( ruleIndex, ruleParseStart ); 540} 541 542template< class ImplTraits> 543ANTLR_INLINE bool Lexer<ImplTraits>::haveParsedRule(ANTLR_MARKER ruleIndex) 544{ 545 return this->get_rec()->alreadyParsedRule(ruleIndex); 546} 547 548template< class ImplTraits> 549ANTLR_INLINE void Lexer<ImplTraits>::setText( const StringType& text ) 550{ 551 this->get_lexstate()->set_text(text); 552} 553 554template< class ImplTraits> 555ANTLR_INLINE void Lexer<ImplTraits>::skip() 556{ 557 CommonTokenType& skipToken = this->get_tokSource()->get_skipToken(); 558 this->get_lexstate()->set_token( &skipToken ); 559} 560 561template< class ImplTraits> 562ANTLR_INLINE typename Lexer<ImplTraits>::RuleMemoType* Lexer<ImplTraits>::getRuleMemo() const 563{ 564 return this->get_lexstate()->get_rulememo(); 565} 566 567template< class ImplTraits> 568ANTLR_INLINE void Lexer<ImplTraits>::setRuleMemo(RuleMemoType* rulememo) 569{ 570 return this->get_lexstate()->set_rulememo(rulememo); 571} 572 573template< class ImplTraits> 574ANTLR_INLINE typename Lexer<ImplTraits>::DebuggerType* Lexer<ImplTraits>::get_debugger() const 575{ 576 return this->get_rec()->get_debugger(); 577} 578 579template< class ImplTraits> 580ANTLR_INLINE ANTLR_UINT32 Lexer<ImplTraits>::LA(ANTLR_INT32 i) 581{ 582 return this->get_istream()->_LA(i); 583} 584 585template< class ImplTraits> 586ANTLR_INLINE void Lexer<ImplTraits>::consume() 587{ 588 return this->get_istream()->consume(); 589} 590 591ANTLR_END_NAMESPACE() 592 593