1 // Copyright (c) 2001-2010 Hartmut Kaiser 2 // 3 // Distributed under the Boost Software License, Version 1.0. (See accompanying 4 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5 6 #if !defined(SPIRIT_LEXER_EXAMPLE_WORD_COUNT_LEXER_TOKENS_FEB_10_2008_0739PM) 7 #define SPIRIT_LEXER_EXAMPLE_WORD_COUNT_LEXER_TOKENS_FEB_10_2008_0739PM 8 9 #include <boost/spirit/include/phoenix_operator.hpp> 10 #include <boost/spirit/include/phoenix_statement.hpp> 11 #include <boost/spirit/include/phoenix_core.hpp> 12 #include <boost/iterator/iterator_traits.hpp> 13 14 /////////////////////////////////////////////////////////////////////////////// 15 // Token definition: We use the lexertl based lexer engine as the underlying 16 // lexer type. 17 // 18 // Note, the token definition type is derived from the 'lexertl_actor_lexer' 19 // template, which is a necessary to being able to use lexer semantic actions. 20 /////////////////////////////////////////////////////////////////////////////// 21 struct distance_func 22 { 23 template <typename Iterator1, typename Iterator2> 24 struct result : boost::iterator_difference<Iterator1> {}; 25 26 template <typename Iterator1, typename Iterator2> 27 typename result<Iterator1, Iterator2>::type operator ()distance_func28 operator()(Iterator1& begin, Iterator2& end) const 29 { 30 return std::distance(begin, end); 31 } 32 }; 33 boost::phoenix::function<distance_func> const distance = distance_func(); 34 35 //[wcl_static_token_definition 36 template <typename Lexer> 37 struct word_count_lexer_tokens : boost::spirit::lex::lexer<Lexer> 38 { word_count_lexer_tokensword_count_lexer_tokens39 word_count_lexer_tokens() 40 : c(0), w(0), l(0) 41 , word("[^ \t\n]+") // define tokens 42 , eol("\n") 43 , any(".") 44 { 45 using boost::spirit::lex::_start; 46 using boost::spirit::lex::_end; 47 using boost::phoenix::ref; 48 49 // associate tokens with the lexer 50 this->self 51 = word [++ref(w), ref(c) += distance(_start, _end)] 52 | eol [++ref(c), ++ref(l)] 53 | any [++ref(c)] 54 ; 55 } 56 57 std::size_t c, w, l; 58 boost::spirit::lex::token_def<> word, eol, any; 59 }; 60 //] 61 62 #endif 63