1 // Copyright (c) 2001-2011 Hartmut Kaiser 2 // 3 // Distributed under the Boost Software License, Version 1.0. (See accompanying 4 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5 6 #if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM) 7 #define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM 8 9 #if defined(_MSC_VER) 10 #pragma once 11 #endif 12 13 #include <boost/config.hpp> 14 #include <boost/detail/workaround.hpp> 15 #include <boost/spirit/home/qi/detail/assign_to.hpp> 16 #include <boost/spirit/home/support/attributes.hpp> 17 #include <boost/spirit/home/support/argument.hpp> 18 #include <boost/spirit/home/support/detail/lexer/generator.hpp> 19 #include <boost/spirit/home/support/detail/lexer/rules.hpp> 20 #include <boost/spirit/home/support/detail/lexer/consts.hpp> 21 #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp> 22 #include <boost/spirit/home/lex/lexer/terminals.hpp> 23 #include <boost/fusion/include/vector.hpp> 24 #include <boost/fusion/include/at.hpp> 25 #include <boost/fusion/include/value_at.hpp> 26 #include <boost/variant.hpp> 27 #include <boost/mpl/vector.hpp> 28 #include <boost/mpl/bool.hpp> 29 #include <boost/mpl/is_sequence.hpp> 30 #include <boost/mpl/begin.hpp> 31 #include <boost/mpl/insert.hpp> 32 #include <boost/mpl/vector.hpp> 33 #include <boost/mpl/if.hpp> 34 #include <boost/mpl/or.hpp> 35 #include <boost/type_traits/is_same.hpp> 36 #include <boost/range/iterator_range_core.hpp> 37 #include <boost/static_assert.hpp> 38 39 #if defined(BOOST_SPIRIT_DEBUG) 40 #include <iosfwd> 41 #endif 42 43 namespace boost { namespace spirit { namespace lex { namespace lexertl 44 { 45 /////////////////////////////////////////////////////////////////////////// 46 // 47 // The position_token is the type of the objects returned by the 48 // iterator if it has been specified while instantiating the lexer object. 49 // 50 // template parameters: 51 // Iterator The type of the iterator used to access the 52 // underlying character stream. 53 // AttributeTypes A mpl sequence containing the types of all 54 // required different token values to be supported 55 // by this token type. 56 // HasState A mpl::bool_ indicating, whether this token type 57 // should support lexer states. 58 // Idtype The type to use for the token id (defaults to 59 // std::size_t). 60 // 61 // It is possible to use other token types with the spirit::lex 62 // framework as well. If you plan to use a different type as your token 63 // type, you'll need to expose the following things from your token type 64 // to make it compatible with spirit::lex: 65 // 66 // typedefs 67 // iterator_type The type of the iterator used to access the 68 // underlying character stream. 69 // 70 // id_type The type of the token id used. 71 // 72 // methods 73 // default constructor 74 // This should initialize the token as an end of 75 // input token. 76 // constructors The prototype of the other required 77 // constructors should be: 78 // 79 // token(int) 80 // This constructor should initialize the token as 81 // an invalid token (not carrying any specific 82 // values) 83 // 84 // where: the int is used as a tag only and its value is 85 // ignored 86 // 87 // and: 88 // 89 // token(Idtype id, std::size_t state, 90 // iterator_type first, iterator_type last); 91 // 92 // where: id: token id 93 // state: lexer state this token was matched in 94 // first, last: pair of iterators marking the matched 95 // range in the underlying input stream 96 // 97 // accessors 98 // id() return the token id of the matched input sequence 99 // id(newid) set the token id of the token instance 100 // 101 // state() return the lexer state this token was matched in 102 // 103 // value() return the token value 104 // 105 // Additionally, you will have to implement a couple of helper functions 106 // in the same namespace as the token type: a comparison operator==() to 107 // compare your token instances, a token_is_valid() function and different 108 // specializations of the Spirit customization point 109 // assign_to_attribute_from_value as shown below. 110 // 111 /////////////////////////////////////////////////////////////////////////// 112 template <typename Iterator = char const* 113 , typename AttributeTypes = mpl::vector0<> 114 , typename HasState = mpl::true_ 115 , typename Idtype = std::size_t> 116 struct position_token; 117 118 /////////////////////////////////////////////////////////////////////////// 119 // This specialization of the token type doesn't contain any item data and 120 // doesn't support working with lexer states. Although, like all other 121 // variants of position_token, it carries a pair of iterators marking the 122 // begin and the end of the matched character sequence. 123 /////////////////////////////////////////////////////////////////////////// 124 template <typename Iterator, typename Idtype> 125 struct position_token<Iterator, lex::omit, mpl::false_, Idtype> 126 { 127 typedef Iterator iterator_type; 128 typedef iterator_range<iterator_type> iterpair_type; 129 typedef mpl::false_ has_state; 130 typedef Idtype id_type; 131 typedef unused_type token_value_type; 132 133 // default constructed tokens correspond to EOI tokens position_tokenboost::spirit::lex::lexertl::position_token134 position_token() 135 : id_(id_type(boost::lexer::npos)) {} 136 137 // construct an invalid token position_tokenboost::spirit::lex::lexertl::position_token138 explicit position_token(int) 139 : id_(id_type(0)) {} 140 position_tokenboost::spirit::lex::lexertl::position_token141 position_token(id_type id, std::size_t) 142 : id_(id) {} 143 position_tokenboost::spirit::lex::lexertl::position_token144 position_token(id_type id, std::size_t, token_value_type) 145 : id_(id) {} 146 position_tokenboost::spirit::lex::lexertl::position_token147 position_token(id_type id, std::size_t, Iterator const& first 148 , Iterator const& last) 149 : id_(id), matched_(first, last) {} 150 151 // this default conversion operator is needed to allow the direct 152 // usage of tokens in conjunction with the primitive parsers defined 153 // in Qi operator id_typeboost::spirit::lex::lexertl::position_token154 operator id_type() const { return id_; } 155 156 // Retrieve or set the token id of this token instance. idboost::spirit::lex::lexertl::position_token157 id_type id() const { return id_; } idboost::spirit::lex::lexertl::position_token158 void id(id_type newid) { id_ = newid; } 159 stateboost::spirit::lex::lexertl::position_token160 std::size_t state() const { return 0; } // always '0' (INITIAL state) 161 is_validboost::spirit::lex::lexertl::position_token162 bool is_valid() const 163 { 164 return 0 != id_ && id_type(boost::lexer::npos) != id_; 165 } 166 167 // access the stored iterator range of the matched input sequence beginboost::spirit::lex::lexertl::position_token168 iterator_type begin() const { return matched_.begin(); } endboost::spirit::lex::lexertl::position_token169 iterator_type end() const { return matched_.end(); } 170 matchedboost::spirit::lex::lexertl::position_token171 iterpair_type& matched() { return matched_; } matchedboost::spirit::lex::lexertl::position_token172 iterpair_type const& matched() const { return matched_; } 173 valueboost::spirit::lex::lexertl::position_token174 token_value_type& value() { static token_value_type u; return u; } valueboost::spirit::lex::lexertl::position_token175 token_value_type const& value() const { return unused; } 176 177 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) 178 // workaround for MSVC10 which has problems copying a default 179 // constructed iterator_range operator =boost::spirit::lex::lexertl::position_token180 position_token& operator= (position_token const& rhs) 181 { 182 if (this != &rhs) 183 { 184 id_ = rhs.id_; 185 if (is_valid()) 186 matched_ = rhs.matched_; 187 } 188 return *this; 189 } 190 #endif 191 192 protected: 193 id_type id_; // token id, 0 if nothing has been matched 194 iterpair_type matched_; // matched input sequence 195 }; 196 197 #if defined(BOOST_SPIRIT_DEBUG) 198 template <typename Char, typename Traits, typename Iterator 199 , typename AttributeTypes, typename HasState, typename Idtype> 200 inline std::basic_ostream<Char, Traits>& operator <<(std::basic_ostream<Char,Traits> & os,position_token<Iterator,AttributeTypes,HasState,Idtype> const & t)201 operator<< (std::basic_ostream<Char, Traits>& os 202 , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) 203 { 204 if (t.is_valid()) { 205 Iterator end = t.end(); 206 for (Iterator it = t.begin(); it != end; ++it) 207 os << *it; 208 } 209 else { 210 os << "<invalid token>"; 211 } 212 return os; 213 } 214 #endif 215 216 /////////////////////////////////////////////////////////////////////////// 217 // This specialization of the token type doesn't contain any item data but 218 // supports working with lexer states. 219 /////////////////////////////////////////////////////////////////////////// 220 template <typename Iterator, typename Idtype> 221 struct position_token<Iterator, lex::omit, mpl::true_, Idtype> 222 : position_token<Iterator, lex::omit, mpl::false_, Idtype> 223 { 224 private: 225 typedef position_token<Iterator, lex::omit, mpl::false_, Idtype> 226 base_type; 227 228 public: 229 typedef typename base_type::id_type id_type; 230 typedef Iterator iterator_type; 231 typedef mpl::true_ has_state; 232 typedef unused_type token_value_type; 233 234 // default constructed tokens correspond to EOI tokens position_tokenboost::spirit::lex::lexertl::position_token235 position_token() : state_(boost::lexer::npos) {} 236 237 // construct an invalid token position_tokenboost::spirit::lex::lexertl::position_token238 explicit position_token(int) 239 : base_type(0), state_(boost::lexer::npos) {} 240 position_tokenboost::spirit::lex::lexertl::position_token241 position_token(id_type id, std::size_t state) 242 : base_type(id, boost::lexer::npos), state_(state) {} 243 position_tokenboost::spirit::lex::lexertl::position_token244 position_token(id_type id, std::size_t state, token_value_type) 245 : base_type(id, boost::lexer::npos, unused) 246 , state_(state) {} 247 position_tokenboost::spirit::lex::lexertl::position_token248 position_token(id_type id, std::size_t state 249 , Iterator const& first, Iterator const& last) 250 : base_type(id, boost::lexer::npos, first, last) 251 , state_(state) {} 252 stateboost::spirit::lex::lexertl::position_token253 std::size_t state() const { return state_; } 254 255 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) 256 // workaround for MSVC10 which has problems copying a default 257 // constructed iterator_range operator =boost::spirit::lex::lexertl::position_token258 position_token& operator= (position_token const& rhs) 259 { 260 if (this != &rhs) 261 { 262 this->base_type::operator=(static_cast<base_type const&>(rhs)); 263 state_ = rhs.state_; 264 } 265 return *this; 266 } 267 #endif 268 269 protected: 270 std::size_t state_; // lexer state this token was matched in 271 }; 272 273 /////////////////////////////////////////////////////////////////////////// 274 // These specializations for an empty attribute list cause all token 275 // instances to expose as it attribute the iterator_range pointing to the 276 // matched input sequence. 277 /////////////////////////////////////////////////////////////////////////// 278 template <typename Iterator, typename HasState, typename Idtype> 279 struct position_token<Iterator, mpl::vector<>, HasState, Idtype> 280 : position_token<Iterator, lex::omit, HasState, Idtype> 281 { 282 private: 283 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; 284 285 public: 286 typedef typename base_type::id_type id_type; 287 typedef typename base_type::iterator_type iterator_type; 288 typedef typename base_type::iterpair_type iterpair_type; 289 typedef HasState has_state; 290 typedef iterpair_type token_value_type; 291 292 // default constructed tokens correspond to EOI tokens 293 position_token() {} 294 295 // construct an invalid token 296 explicit position_token(int) 297 : base_type(0) {} 298 299 position_token(id_type id, std::size_t state) 300 : base_type(id, state) {} 301 302 position_token(id_type id, std::size_t state, token_value_type) 303 : base_type(id, state, unused) {} 304 305 position_token(id_type id, std::size_t state 306 , Iterator const& first, Iterator const& last) 307 : base_type(id, state, first, last) {} 308 309 token_value_type& value() { return this->base_type::matched(); } 310 token_value_type const& value() const { return this->base_type::matched(); } 311 }; 312 313 template <typename Iterator, typename HasState, typename Idtype> 314 struct position_token<Iterator, mpl::vector0<>, HasState, Idtype> 315 : position_token<Iterator, lex::omit, HasState, Idtype> 316 { 317 private: 318 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; 319 320 public: 321 typedef typename base_type::id_type id_type; 322 typedef typename base_type::iterator_type iterator_type; 323 typedef typename base_type::iterpair_type iterpair_type; 324 typedef HasState has_state; 325 typedef iterpair_type token_value_type; 326 327 // default constructed tokens correspond to EOI tokens 328 position_token() {} 329 330 // construct an invalid token 331 explicit position_token(int) 332 : base_type(0) {} 333 334 position_token(id_type id, std::size_t state) 335 : base_type(id, state) {} 336 337 position_token(id_type id, std::size_t state, token_value_type) 338 : base_type(id, state, unused) {} 339 340 position_token(id_type id, std::size_t state 341 , Iterator const& first, Iterator const& last) 342 : base_type(id, state, first, last) {} 343 344 token_value_type& value() { return this->base_type::matched(); } 345 token_value_type const& value() const { return this->base_type::matched(); } 346 }; 347 348 /////////////////////////////////////////////////////////////////////////// 349 // These specializations for an attribute list of length one cause all token 350 // instances to expose the specified type as its attribute. 351 /////////////////////////////////////////////////////////////////////////// 352 template <typename Iterator, typename Attribute, typename HasState 353 , typename Idtype> 354 struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype> 355 : position_token<Iterator, lex::omit, HasState, Idtype> 356 { 357 private: 358 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; 359 360 public: 361 typedef typename base_type::id_type id_type; 362 typedef typename base_type::iterator_type iterator_type; 363 typedef typename base_type::iterpair_type iterpair_type; 364 typedef HasState has_state; 365 typedef boost::optional<Attribute> token_value_type; 366 367 // default constructed tokens correspond to EOI tokens position_tokenboost::spirit::lex::lexertl::position_token368 position_token() {} 369 370 // construct an invalid token position_tokenboost::spirit::lex::lexertl::position_token371 explicit position_token(int) 372 : base_type(0) {} 373 position_tokenboost::spirit::lex::lexertl::position_token374 position_token(id_type id, std::size_t state) 375 : base_type(id, state) {} 376 position_tokenboost::spirit::lex::lexertl::position_token377 position_token(id_type id, std::size_t state, token_value_type const& v) 378 : base_type(id, state, unused), value_(v) {} 379 position_tokenboost::spirit::lex::lexertl::position_token380 position_token(id_type id, std::size_t state 381 , Iterator const& first, Iterator const& last) 382 : base_type(id, state, first, last) {} 383 valueboost::spirit::lex::lexertl::position_token384 token_value_type& value() { return value_; } valueboost::spirit::lex::lexertl::position_token385 token_value_type const& value() const { return value_; } 386 has_valueboost::spirit::lex::lexertl::position_token387 bool has_value() const { return !!value_; } 388 389 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) 390 // workaround for MSVC10 which has problems copying a default 391 // constructed iterator_range operator =boost::spirit::lex::lexertl::position_token392 position_token& operator= (position_token const& rhs) 393 { 394 if (this != &rhs) 395 { 396 this->base_type::operator=(static_cast<base_type const&>(rhs)); 397 if (this->is_valid()) 398 value_ = rhs.value_; 399 } 400 return *this; 401 } 402 #endif 403 404 protected: 405 token_value_type value_; // token value 406 }; 407 408 template <typename Iterator, typename Attribute, typename HasState 409 , typename Idtype> 410 struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype> 411 : position_token<Iterator, lex::omit, HasState, Idtype> 412 { 413 private: 414 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type; 415 416 public: 417 typedef typename base_type::id_type id_type; 418 typedef typename base_type::iterator_type iterator_type; 419 typedef typename base_type::iterpair_type iterpair_type; 420 typedef HasState has_state; 421 typedef boost::optional<Attribute> token_value_type; 422 423 // default constructed tokens correspond to EOI tokens position_tokenboost::spirit::lex::lexertl::position_token424 position_token() {} 425 426 // construct an invalid token position_tokenboost::spirit::lex::lexertl::position_token427 explicit position_token(int) 428 : base_type(0) {} 429 position_tokenboost::spirit::lex::lexertl::position_token430 position_token(id_type id, std::size_t state) 431 : base_type(id, state) {} 432 position_tokenboost::spirit::lex::lexertl::position_token433 position_token(id_type id, std::size_t state, token_value_type const& v) 434 : base_type(id, state, unused), value_(v) {} 435 position_tokenboost::spirit::lex::lexertl::position_token436 position_token(id_type id, std::size_t state 437 , Iterator const& first, Iterator const& last) 438 : base_type(id, state, first, last) {} 439 valueboost::spirit::lex::lexertl::position_token440 token_value_type& value() { return value_; } valueboost::spirit::lex::lexertl::position_token441 token_value_type const& value() const { return value_; } 442 has_valueboost::spirit::lex::lexertl::position_token443 bool has_value() const { return value_; } 444 445 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) 446 // workaround for MSVC10 which has problems copying a default 447 // constructed iterator_range operator =boost::spirit::lex::lexertl::position_token448 position_token& operator= (position_token const& rhs) 449 { 450 if (this != &rhs) 451 { 452 this->base_type::operator=(static_cast<base_type const&>(rhs)); 453 if (this->is_valid()) 454 value_ = rhs.value_; 455 } 456 return *this; 457 } 458 #endif 459 460 protected: 461 token_value_type value_; // token value 462 }; 463 464 /////////////////////////////////////////////////////////////////////////// 465 // The generic version of the position_token type derives from the 466 // specialization above and adds a single data member holding the item 467 // data carried by the token instance. 468 /////////////////////////////////////////////////////////////////////////// 469 namespace detail 470 { 471 /////////////////////////////////////////////////////////////////////// 472 // Meta-function to calculate the type of the variant data item to be 473 // stored with each token instance. 474 // 475 // Note: The iterator pair needs to be the first type in the list of 476 // types supported by the generated variant type (this is being 477 // used to identify whether the stored data item in a particular 478 // token instance needs to be converted from the pair of 479 // iterators (see the first of the assign_to_attribute_from_value 480 // specializations below). 481 /////////////////////////////////////////////////////////////////////// 482 template <typename IteratorPair, typename AttributeTypes> 483 struct position_token_value_typesequence 484 { 485 typedef typename mpl::insert< 486 AttributeTypes 487 , typename mpl::begin<AttributeTypes>::type 488 , IteratorPair 489 >::type sequence_type; 490 typedef typename make_variant_over<sequence_type>::type type; 491 }; 492 493 /////////////////////////////////////////////////////////////////////// 494 // The type of the data item stored with a token instance is defined 495 // by the template parameter 'AttributeTypes' and may be: 496 // 497 // lex::omit: no data item is stored with the token 498 // instance (this is handled by the 499 // specializations of the token class 500 // below) 501 // mpl::vector0<>: each token instance stores a pair of 502 // iterators pointing to the matched input 503 // sequence 504 // mpl::vector<...>: each token instance stores a variant being 505 // able to store the pair of iterators pointing 506 // to the matched input sequence, or any of the 507 // types a specified in the mpl::vector<> 508 // 509 // All this is done to ensure the token type is as small (in terms 510 // of its byte-size) as possible. 511 /////////////////////////////////////////////////////////////////////// 512 template <typename IteratorPair, typename AttributeTypes> 513 struct position_token_value 514 : mpl::eval_if< 515 mpl::or_< 516 is_same<AttributeTypes, mpl::vector0<> > 517 , is_same<AttributeTypes, mpl::vector<> > > 518 , mpl::identity<IteratorPair> 519 , position_token_value_typesequence<IteratorPair, AttributeTypes> > 520 {}; 521 } 522 523 template <typename Iterator, typename AttributeTypes, typename HasState 524 , typename Idtype> 525 struct position_token 526 : position_token<Iterator, lex::omit, HasState, Idtype> 527 { 528 private: // precondition assertions 529 BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || 530 is_same<AttributeTypes, lex::omit>::value)); 531 typedef position_token<Iterator, lex::omit, HasState, Idtype> 532 base_type; 533 534 protected: 535 // If no additional token value types are given, the token will 536 // hold no token value at all as the base class already has the 537 // iterator pair of the matched range in the underlying input sequence. 538 // Otherwise the token value is stored as a variant and will 539 // initially hold an unused_type but is able to hold any of 540 // the given data types as well. The conversion from the iterator pair 541 // to the required data type is done when it is accessed for the first 542 // time. 543 typedef iterator_range<Iterator> iterpair_type; 544 545 public: 546 typedef typename base_type::id_type id_type; 547 typedef typename detail::position_token_value< 548 iterpair_type, AttributeTypes>::type token_value_type; 549 550 typedef Iterator iterator_type; 551 552 // default constructed tokens correspond to EOI tokens position_tokenboost::spirit::lex::lexertl::position_token553 position_token() {} 554 555 // construct an invalid token position_tokenboost::spirit::lex::lexertl::position_token556 explicit position_token(int) 557 : base_type(0) {} 558 position_tokenboost::spirit::lex::lexertl::position_token559 position_token(id_type id, std::size_t state, token_value_type const& value) 560 : base_type(id, state, value), value_(value) {} 561 position_tokenboost::spirit::lex::lexertl::position_token562 position_token(id_type id, std::size_t state, Iterator const& first 563 , Iterator const& last) 564 : base_type(id, state, first, last) 565 , value_(iterpair_type(first, last)) 566 {} 567 valueboost::spirit::lex::lexertl::position_token568 token_value_type& value() { return value_; } valueboost::spirit::lex::lexertl::position_token569 token_value_type const& value() const { return value_; } 570 571 #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) 572 // workaround for MSVC10 which has problems copying a default 573 // constructed iterator_range operator =boost::spirit::lex::lexertl::position_token574 position_token& operator= (position_token const& rhs) 575 { 576 if (this != &rhs) 577 { 578 this->base_type::operator=(static_cast<base_type const&>(rhs)); 579 if (this->is_valid()) 580 value_ = rhs.value_; 581 } 582 return *this; 583 } 584 #endif 585 586 protected: 587 token_value_type value_; // token value, by default a pair of iterators 588 }; 589 590 /////////////////////////////////////////////////////////////////////////// 591 // tokens are considered equal, if their id's match (these are unique) 592 template <typename Iterator, typename AttributeTypes, typename HasState 593 , typename Idtype> 594 inline bool operator ==(position_token<Iterator,AttributeTypes,HasState,Idtype> const & lhs,position_token<Iterator,AttributeTypes,HasState,Idtype> const & rhs)595 operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs, 596 position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs) 597 { 598 return lhs.id() == rhs.id(); 599 } 600 601 /////////////////////////////////////////////////////////////////////////// 602 // This overload is needed by the multi_pass/functor_input_policy to 603 // validate a token instance. It has to be defined in the same namespace 604 // as the token class itself to allow ADL to find it. 605 /////////////////////////////////////////////////////////////////////////// 606 template <typename Iterator, typename AttributeTypes, typename HasState 607 , typename Idtype> 608 inline bool token_is_valid(position_token<Iterator,AttributeTypes,HasState,Idtype> const & t)609 token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t) 610 { 611 return t.is_valid(); 612 } 613 }}}} 614 615 namespace boost { namespace spirit { namespace traits 616 { 617 /////////////////////////////////////////////////////////////////////////// 618 // We have to provide specializations for the customization point 619 // assign_to_attribute_from_value allowing to extract the needed value 620 // from the token. 621 /////////////////////////////////////////////////////////////////////////// 622 623 // This is called from the parse function of token_def if the token_def 624 // has been defined to carry a special attribute type 625 template <typename Attribute, typename Iterator, typename AttributeTypes 626 , typename HasState, typename Idtype> 627 struct assign_to_attribute_from_value<Attribute 628 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 629 { 630 static void callboost::spirit::traits::assign_to_attribute_from_value631 call(lex::lexertl::position_token< 632 Iterator, AttributeTypes, HasState, Idtype> const& t 633 , Attribute& attr) 634 { 635 // The goal of this function is to avoid the conversion of the pair of 636 // iterators (to the matched character sequence) into the token value 637 // of the required type being done more than once. For this purpose it 638 // checks whether the stored value type is still the default one (pair 639 // of iterators) and if yes, replaces the pair of iterators with the 640 // converted value to be returned from subsequent calls. 641 642 if (0 == t.value().which()) { 643 // first access to the token value 644 typedef iterator_range<Iterator> iterpair_type; 645 iterpair_type const& ip = t.matched(); 646 647 // Interestingly enough we use the assign_to() framework defined in 648 // Spirit.Qi allowing to convert the pair of iterators to almost any 649 // required type (assign_to(), if available, uses the standard Spirit 650 // parsers to do the conversion). 651 spirit::traits::assign_to(ip.begin(), ip.end(), attr); 652 653 // If you get an error during the compilation of the following 654 // assignment expression, you probably forgot to list one or more 655 // types used as token value types (in your token_def<...> 656 // definitions) in your definition of the token class. I.e. any token 657 // value type used for a token_def<...> definition has to be listed 658 // during the declaration of the token type to use. For instance let's 659 // assume we have two token_def's: 660 // 661 // token_def<int> number; number = "..."; 662 // token_def<std::string> identifier; identifier = "..."; 663 // 664 // Then you'll have to use the following token type definition 665 // (assuming you are using the token class): 666 // 667 // typedef mpl::vector<int, std::string> token_values; 668 // typedef token<base_iter_type, token_values> token_type; 669 // 670 // where: base_iter_type is the iterator type used to expose the 671 // underlying input stream. 672 // 673 // This token_type has to be used as the second template parameter 674 // to the lexer class: 675 // 676 // typedef lexer<base_iter_type, token_type> lexer_type; 677 // 678 // again, assuming you're using the lexer<> template for your 679 // tokenization. 680 681 typedef lex::lexertl::position_token< 682 Iterator, AttributeTypes, HasState, Idtype> token_type; 683 spirit::traits::assign_to( 684 attr, const_cast<token_type&>(t).value()); // re-assign value 685 } 686 else { 687 // reuse the already assigned value 688 spirit::traits::assign_to(get<Attribute>(t.value()), attr); 689 } 690 } 691 }; 692 693 template <typename Attribute, typename Iterator, typename AttributeTypes 694 , typename HasState, typename Idtype> 695 struct assign_to_container_from_value<Attribute 696 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 697 : assign_to_attribute_from_value<Attribute 698 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 699 {}; 700 701 /////////////////////////////////////////////////////////////////////////// 702 // These are called from the parse function of token_def if the token type 703 // has no special attribute type assigned 704 template <typename Attribute, typename Iterator, typename HasState 705 , typename Idtype> 706 struct assign_to_attribute_from_value<Attribute 707 , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > 708 { 709 static void 710 call(lex::lexertl::position_token< 711 Iterator, mpl::vector0<>, HasState, Idtype> const& t 712 , Attribute& attr) 713 { 714 // The default type returned by the token_def parser component (if 715 // it has no token value type assigned) is the pair of iterators 716 // to the matched character sequence. 717 spirit::traits::assign_to(t.begin(), t.end(), attr); 718 } 719 }; 720 721 // template <typename Attribute, typename Iterator, typename HasState 722 // , typename Idtype> 723 // struct assign_to_container_from_value<Attribute 724 // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > 725 // : assign_to_attribute_from_value<Attribute 726 // , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> > 727 // {}; 728 729 // same as above but using mpl::vector<> instead of mpl::vector0<> 730 template <typename Attribute, typename Iterator, typename HasState 731 , typename Idtype> 732 struct assign_to_attribute_from_value<Attribute 733 , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > 734 { 735 static void 736 call(lex::lexertl::position_token< 737 Iterator, mpl::vector<>, HasState, Idtype> const& t 738 , Attribute& attr) 739 { 740 // The default type returned by the token_def parser component (if 741 // it has no token value type assigned) is the pair of iterators 742 // to the matched character sequence. 743 spirit::traits::assign_to(t.begin(), t.end(), attr); 744 } 745 }; 746 747 // template <typename Attribute, typename Iterator, typename HasState 748 // , typename Idtype> 749 // struct assign_to_container_from_value<Attribute 750 // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > 751 // : assign_to_attribute_from_value<Attribute 752 // , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> > 753 // {}; 754 755 /////////////////////////////////////////////////////////////////////////// 756 // These are called from the parse function of token_def if the token type 757 // has no special attribute type assigned 758 template <typename Attribute, typename Iterator, typename Attr 759 , typename HasState, typename Idtype> 760 struct assign_to_attribute_from_value<Attribute 761 , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > 762 { 763 static void callboost::spirit::traits::assign_to_attribute_from_value764 call(lex::lexertl::position_token< 765 Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t 766 , Attribute& attr) 767 { 768 // The goal of this function is to avoid the conversion of the pair of 769 // iterators (to the matched character sequence) into the token value 770 // of the required type being done more than once. 771 772 if (!t.has_value()) { 773 // first access to the token value 774 typedef iterator_range<Iterator> iterpair_type; 775 iterpair_type const& ip = t.matched(); 776 777 // Interestingly enough we use the assign_to() framework defined in 778 // Spirit.Qi allowing to convert the pair of iterators to almost any 779 // required type (assign_to(), if available, uses the standard Spirit 780 // parsers to do the conversion). 781 spirit::traits::assign_to(ip.begin(), ip.end(), attr); 782 783 // Re-assign the attribute to the stored value 784 typedef lex::lexertl::position_token< 785 Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type; 786 spirit::traits::assign_to( 787 attr, const_cast<token_type&>(t).value()); 788 } 789 else { 790 // reuse the already assigned value 791 spirit::traits::assign_to(t.value(), attr); 792 } 793 } 794 }; 795 796 // template <typename Attribute, typename Iterator, typename Attr 797 // , typename HasState, typename Idtype> 798 // struct assign_to_container_from_value<Attribute 799 // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > 800 // : assign_to_attribute_from_value<Attribute 801 // , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> > 802 // {}; 803 804 // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr> 805 template <typename Attribute, typename Iterator, typename Attr 806 , typename HasState, typename Idtype> 807 struct assign_to_attribute_from_value<Attribute 808 , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > 809 { 810 static void callboost::spirit::traits::assign_to_attribute_from_value811 call(lex::lexertl::position_token< 812 Iterator, mpl::vector<Attr>, HasState, Idtype> const& t 813 , Attribute& attr) 814 { 815 // The goal of this function is to avoid the conversion of the pair of 816 // iterators (to the matched character sequence) into the token value 817 // of the required type being done more than once. 818 819 if (!t.has_value()) { 820 // first access to the token value 821 typedef iterator_range<Iterator> iterpair_type; 822 iterpair_type const& ip = t.matched(); 823 824 // Interestingly enough we use the assign_to() framework defined in 825 // Spirit.Qi allowing to convert the pair of iterators to almost any 826 // required type (assign_to(), if available, uses the standard Spirit 827 // parsers to do the conversion). 828 spirit::traits::assign_to(ip.begin(), ip.end(), attr); 829 830 // Re-assign the attribute to the stored value 831 typedef lex::lexertl::position_token< 832 Iterator, mpl::vector<Attr>, HasState, Idtype> token_type; 833 spirit::traits::assign_to( 834 attr, const_cast<token_type&>(t).value()); 835 } 836 else { 837 // reuse the already assigned value 838 spirit::traits::assign_to(t.value(), attr); 839 } 840 } 841 }; 842 843 // template <typename Attribute, typename Iterator, typename Attr 844 // , typename HasState, typename Idtype> 845 // struct assign_to_container_from_value<Attribute 846 // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > 847 // : assign_to_attribute_from_value<Attribute 848 // , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> > 849 // {}; 850 851 // This is called from the parse function of token_def if the token type 852 // has been explicitly omitted (i.e. no attribute value is used), which 853 // essentially means that every attribute gets initialized using default 854 // constructed values. 855 template <typename Attribute, typename Iterator, typename HasState 856 , typename Idtype> 857 struct assign_to_attribute_from_value<Attribute 858 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > 859 { 860 static void callboost::spirit::traits::assign_to_attribute_from_value861 call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const& 862 , Attribute&) 863 { 864 // do nothing 865 } 866 }; 867 868 template <typename Attribute, typename Iterator, typename HasState 869 , typename Idtype> 870 struct assign_to_container_from_value<Attribute 871 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > 872 : assign_to_attribute_from_value<Attribute 873 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> > 874 {}; 875 876 // This is called from the parse function of lexer_def_ 877 template <typename Iterator, typename AttributeTypes, typename HasState 878 , typename Idtype_, typename Idtype> 879 struct assign_to_attribute_from_value< 880 fusion::vector2<Idtype_, iterator_range<Iterator> > 881 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 882 { 883 static void callboost::spirit::traits::assign_to_attribute_from_value884 call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t 885 , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr) 886 { 887 // The type returned by the lexer_def_ parser components is a 888 // fusion::vector containing the token id of the matched token 889 // and the pair of iterators to the matched character sequence. 890 typedef iterator_range<Iterator> iterpair_type; 891 typedef fusion::vector2<Idtype_, iterator_range<Iterator> > 892 attribute_type; 893 894 iterpair_type const& ip = t.matched(); 895 attr = attribute_type(t.id(), ip); 896 } 897 }; 898 899 template <typename Iterator, typename AttributeTypes, typename HasState 900 , typename Idtype_, typename Idtype> 901 struct assign_to_container_from_value< 902 fusion::vector2<Idtype_, iterator_range<Iterator> > 903 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 904 : assign_to_attribute_from_value< 905 fusion::vector2<Idtype_, iterator_range<Iterator> > 906 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> > 907 {}; 908 909 /////////////////////////////////////////////////////////////////////////// 910 // Overload debug output for a single token, this integrates lexer tokens 911 // with Qi's simple_trace debug facilities 912 template <typename Iterator, typename Attribute, typename HasState 913 , typename Idtype> 914 struct token_printer_debug< 915 lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> > 916 { 917 typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type; 918 919 template <typename Out> printboost::spirit::traits::token_printer_debug920 static void print(Out& out, token_type const& val) 921 { 922 out << '['; 923 spirit::traits::print_token(out, val.value()); 924 out << ']'; 925 } 926 }; 927 }}} 928 929 #endif 930