1 // Copyright (c) 2001-2011 Hartmut Kaiser
2 //
3 // Distributed under the Boost Software License, Version 1.0. (See accompanying
4 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6 #include <boost/config/warning_disable.hpp>
7 #include <boost/detail/lightweight_test.hpp>
8
9 #include <boost/spirit/include/lex_lexertl.hpp>
10 #include <boost/spirit/include/lex_lexertl_position_token.hpp>
11 #include <boost/spirit/include/phoenix_object.hpp>
12 #include <boost/spirit/include/phoenix_operator.hpp>
13 #include <boost/spirit/include/phoenix_container.hpp>
14 #include <boost/spirit/include/qi_numeric.hpp>
15
16 namespace spirit = boost::spirit;
17 namespace lex = boost::spirit::lex;
18 namespace phoenix = boost::phoenix;
19 namespace mpl = boost::mpl;
20
21 ///////////////////////////////////////////////////////////////////////////////
22 enum tokenids
23 {
24 ID_INT = 1000,
25 ID_DOUBLE
26 };
27
28 template <typename Lexer>
29 struct token_definitions : lex::lexer<Lexer>
30 {
token_definitionstoken_definitions31 token_definitions()
32 {
33 this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
34 this->self.add_pattern("OCTALDIGIT", "[0-7]");
35 this->self.add_pattern("DIGIT", "[0-9]");
36
37 this->self.add_pattern("OPTSIGN", "[-+]?");
38 this->self.add_pattern("EXPSTART", "[eE][-+]");
39 this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
40
41 // define tokens and associate them with the lexer
42 int_ = "{OPTSIGN}[1-9]{DIGIT}*";
43 int_.id(ID_INT);
44
45 double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
46 double_.id(ID_DOUBLE);
47
48 whitespace = "[ \t\n]+";
49
50 this->self =
51 double_
52 | int_
53 | whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
54 ;
55 }
56
57 lex::token_def<double> int_;
58 lex::token_def<double> double_;
59 lex::token_def<lex::omit> whitespace;
60 };
61
62 template <typename Lexer>
63 struct token_definitions_with_state : lex::lexer<Lexer>
64 {
token_definitions_with_statetoken_definitions_with_state65 token_definitions_with_state()
66 {
67 this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
68 this->self.add_pattern("OCTALDIGIT", "[0-7]");
69 this->self.add_pattern("DIGIT", "[0-9]");
70
71 this->self.add_pattern("OPTSIGN", "[-+]?");
72 this->self.add_pattern("EXPSTART", "[eE][-+]");
73 this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
74
75 this->self.add_state();
76 this->self.add_state("INT");
77 this->self.add_state("DOUBLE");
78
79 // define tokens and associate them with the lexer
80 int_ = "{OPTSIGN}[1-9]{DIGIT}*";
81 int_.id(ID_INT);
82
83 double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
84 double_.id(ID_DOUBLE);
85
86 whitespace = "[ \t\n]+";
87
88 this->self("*") =
89 double_ [ lex::_state = "DOUBLE"]
90 | int_ [ lex::_state = "INT" ]
91 | whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
92 ;
93 }
94
95 lex::token_def<double> int_;
96 lex::token_def<double> double_;
97 lex::token_def<lex::omit> whitespace;
98 };
99
100 ///////////////////////////////////////////////////////////////////////////////
101 template <typename Token>
102 inline bool
test_token_ids(int const * ids,std::vector<Token> const & tokens)103 test_token_ids(int const* ids, std::vector<Token> const& tokens)
104 {
105 BOOST_FOREACH(Token const& t, tokens)
106 {
107 if (*ids == -1)
108 return false; // reached end of expected data
109
110 if (t.id() != static_cast<std::size_t>(*ids)) // token id must match
111 return false;
112
113 ++ids;
114 }
115
116 return (*ids == -1) ? true : false;
117 }
118
119 ///////////////////////////////////////////////////////////////////////////////
120 template <typename Token>
121 inline bool
test_token_states(std::size_t const * states,std::vector<Token> const & tokens)122 test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
123 {
124 BOOST_FOREACH(Token const& t, tokens)
125 {
126 if (*states == std::size_t(-1))
127 return false; // reached end of expected data
128
129 if (t.state() != *states) // token state must match
130 return false;
131
132 ++states;
133 }
134
135 return (*states == std::size_t(-1)) ? true : false;
136 }
137
138 ///////////////////////////////////////////////////////////////////////////////
139 struct position_type
140 {
141 std::size_t begin, end;
142 };
143
144 template <typename Iterator, typename Token>
145 inline bool
test_token_positions(Iterator begin,position_type const * positions,std::vector<Token> const & tokens)146 test_token_positions(Iterator begin, position_type const* positions,
147 std::vector<Token> const& tokens)
148 {
149 BOOST_FOREACH(Token const& t, tokens)
150 {
151 if (positions->begin == std::size_t(-1) &&
152 positions->end == std::size_t(-1))
153 {
154 return false; // reached end of expected data
155 }
156
157 boost::iterator_range<Iterator> matched = t.matched();
158 std::size_t start = std::distance(begin, matched.begin());
159 std::size_t end = std::distance(begin, matched.end());
160
161 // position must match
162 if (start != positions->begin || end != positions->end)
163 return false;
164
165 ++positions;
166 }
167
168 return (positions->begin == std::size_t(-1) &&
169 positions->end == std::size_t(-1)) ? true : false;
170 }
171
172 ///////////////////////////////////////////////////////////////////////////////
173 template <typename Token>
174 inline bool
test_token_values(double const * values,std::vector<Token> const & tokens)175 test_token_values(double const* values, std::vector<Token> const& tokens)
176 {
177 BOOST_FOREACH(Token const& t, tokens)
178 {
179 if (*values == 0.0)
180 return false; // reached end of expected data
181
182 double val;
183 spirit::traits::assign_to(t, val);
184 if (val != *values) // token value must match
185 return false;
186
187 ++values;
188 }
189
190 return (*values == 0.0) ? true : false;
191 }
192
193 ///////////////////////////////////////////////////////////////////////////////
main()194 int main()
195 {
196 typedef std::string::iterator base_iterator_type;
197 std::string input(" 1 1.2 -2 3 2.3e6 -3.4");
198 int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
199 std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
200 position_type positions[] =
201 {
202 { 2, 3 }, { 4, 7 }, { 8, 10 }, { 13, 14 }, { 15, 20 }, { 21, 25 },
203 { std::size_t(-1), std::size_t(-1) }
204 };
205 double values[] = { 1.0, 1.2, -2.0, 3.0, 2.3e6, -3.4, 0.0 };
206
207 // token type: token id, iterator_pair as token value, no state
208 {
209 typedef lex::lexertl::token<
210 base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
211 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
212
213 token_definitions<lexer_type> lexer;
214 std::vector<token_type> tokens;
215 base_iterator_type first = input.begin();
216
217 using phoenix::arg_names::_1;
218 BOOST_TEST(lex::tokenize(first, input.end(), lexer
219 , phoenix::push_back(phoenix::ref(tokens), _1)));
220
221 BOOST_TEST(test_token_ids(ids, tokens));
222 BOOST_TEST(test_token_values(values, tokens));
223 }
224
225 {
226 typedef lex::lexertl::position_token<
227 base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
228 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
229
230 token_definitions<lexer_type> lexer;
231 std::vector<token_type> tokens;
232 base_iterator_type first = input.begin();
233
234 using phoenix::arg_names::_1;
235 BOOST_TEST(lex::tokenize(first, input.end(), lexer
236 , phoenix::push_back(phoenix::ref(tokens), _1)));
237
238 BOOST_TEST(test_token_ids(ids, tokens));
239 BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
240 BOOST_TEST(test_token_values(values, tokens));
241 }
242
243 // token type: holds token id, state, iterator_pair as token value
244 {
245 typedef lex::lexertl::token<
246 base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
247 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
248
249 token_definitions_with_state<lexer_type> lexer;
250 std::vector<token_type> tokens;
251 base_iterator_type first = input.begin();
252
253 using phoenix::arg_names::_1;
254 BOOST_TEST(lex::tokenize(first, input.end(), lexer
255 , phoenix::push_back(phoenix::ref(tokens), _1)));
256
257 BOOST_TEST(test_token_ids(ids, tokens));
258 BOOST_TEST(test_token_states(states, tokens));
259 BOOST_TEST(test_token_values(values, tokens));
260 }
261
262 {
263 typedef lex::lexertl::position_token<
264 base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
265 typedef lex::lexertl::actor_lexer<token_type> lexer_type;
266
267 token_definitions_with_state<lexer_type> lexer;
268 std::vector<token_type> tokens;
269 base_iterator_type first = input.begin();
270
271 using phoenix::arg_names::_1;
272 BOOST_TEST(lex::tokenize(first, input.end(), lexer
273 , phoenix::push_back(phoenix::ref(tokens), _1)));
274
275 BOOST_TEST(test_token_ids(ids, tokens));
276 BOOST_TEST(test_token_states(states, tokens));
277 BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
278 BOOST_TEST(test_token_values(values, tokens));
279 }
280
281 return boost::report_errors();
282 }
283