1 // Copyright (c) 2001-2010 Hartmut Kaiser
2 //
3 // Distributed under the Boost Software License, Version 1.0. (See accompanying
4 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6 // The purpose of this example is to show, how it is possible to use a lexer
7 // token definition for two purposes:
8 //
9 // . To generate C++ code implementing a static lexical analyzer allowing
10 // to recognize all defined tokens
11 // . To integrate the generated C++ lexer into the /Spirit/ framework.
12 //
13
14 // #define BOOST_SPIRIT_DEBUG
15 // #define BOOST_SPIRIT_LEXERTL_DEBUG
16
17 #include <boost/config/warning_disable.hpp>
18 #include <boost/spirit/include/lex_static_lexertl.hpp>
19
20 #include <iostream>
21 #include <string>
22
23 #include "../example.hpp"
24 #include "word_count_lexer_tokens.hpp" // token definition
25 #include "word_count_lexer_static.hpp" // generated tokenizer
26
27 using namespace boost::spirit;
28
29 ///////////////////////////////////////////////////////////////////////////////
30 //[wcl_static_main
main(int argc,char * argv[])31 int main(int argc, char* argv[])
32 {
33 // read input from the given file
34 std::string str (read_from_file(1 == argc ? "word_count.input" : argv[1]));
35
36 // Specifying 'omit' as the token attribute type generates a token class
37 // notholding any token attribute at all (not even the iterator_range of the
38 // matched input sequence), therefor optimizing the token, the lexer, and
39 // possibly the parser implementation as much as possible.
40 //
41 // Specifying mpl::false_ as the 3rd template parameter generates a token
42 // type and an iterator, both holding no lexer state, allowing for even more
43 // aggressive optimizations.
44 //
45 // As a result the token instances contain the token ids as the only data
46 // member.
47 typedef lex::lexertl::token<char const*, lex::omit, boost::mpl::false_> token_type;
48
49 // Define the lexer type to be used as the base class for our token
50 // definition.
51 //
52 // This is the only place where the code is different from an equivalent
53 // dynamic lexical analyzer. We use the `lexertl::static_lexer<>` instead of
54 // the `lexertl::lexer<>` as the base class for our token definition type.
55 //
56 // As we specified the suffix "wcl" while generating the static tables we
57 // need to pass the type lexertl::static_::lexer_wcl as the second template
58 // parameter below (see word_count_lexer_generate.cpp).
59 typedef lex::lexertl::static_actor_lexer<
60 token_type, lex::lexertl::static_::lexer_wcl
61 > lexer_type;
62
63 // create the lexer object instance needed to invoke the lexical analysis
64 word_count_lexer_tokens<lexer_type> word_count_lexer;
65
66 // tokenize the given string, all generated tokens are discarded
67 char const* first = str.c_str();
68 char const* last = &first[str.size()];
69 bool r = lex::tokenize(first, last, word_count_lexer);
70
71 if (r) {
72 std::cout << "lines: " << word_count_lexer.l
73 << ", words: " << word_count_lexer.w
74 << ", characters: " << word_count_lexer.c
75 << "\n";
76 }
77 else {
78 std::string rest(first, last);
79 std::cout << "Lexical analysis failed\n" << "stopped at: \""
80 << rest << "\"\n";
81 }
82 return 0;
83 }
84 //]
85