1 // (C) Copyright Gennadiy Rozental 2004-2008.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
5
6 // See http://www.boost.org/libs/test for the library home page.
7 //
8 // File : $RCSfile$
9 //
10 // Version : $Revision: 54633 $
11 //
12 // Description : token iterator for string and range tokenization
13 // ***************************************************************************
14
15 #ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER
16 #define BOOST_TOKEN_ITERATOR_HPP_071894GER
17
18 // Boost
19 #include <boost/config.hpp>
20 #include <boost/detail/workaround.hpp>
21
22 #include <boost/iterator/iterator_categories.hpp>
23 #include <boost/iterator/iterator_traits.hpp>
24
25 #include <boost/test/utils/iterator/input_iterator_facade.hpp>
26 #include <boost/test/utils/basic_cstring/basic_cstring.hpp>
27 #include <boost/test/utils/named_params.hpp>
28 #include <boost/test/utils/foreach.hpp>
29
30 // STL
31 #include <iosfwd>
32 #include <cctype>
33
34 #include <boost/test/detail/suppress_warnings.hpp>
35
36 //____________________________________________________________________________//
37
38 #ifdef BOOST_NO_STDC_NAMESPACE
39 namespace std{ using ::ispunct; using ::isspace; }
40 #endif
41
42 namespace boost {
43
44 namespace unit_test {
45
46 // ************************************************************************** //
47 // ************** ti_delimeter_type ************** //
48 // ************************************************************************** //
49
50 enum ti_delimeter_type {
51 dt_char, // character is delimeter if it among explicit list of some characters
52 dt_ispunct, // character is delimeter if it satisfies ispunct functor
53 dt_isspace, // character is delimeter if it satisfies isspace functor
54 dt_none // no character is delimeter
55 };
56
57 namespace ut_detail {
58
59 // ************************************************************************** //
60 // ************** default_char_compare ************** //
61 // ************************************************************************** //
62
63 template<typename CharT>
64 class default_char_compare {
65 public:
operator ()(CharT c1,CharT c2)66 bool operator()( CharT c1, CharT c2 )
67 {
68 #ifdef BOOST_CLASSIC_IOSTREAMS
69 return std::string_char_traits<CharT>::eq( c1, c2 );
70 #else
71 return std::char_traits<CharT>::eq( c1, c2 );
72 #endif
73 }
74 };
75
76 // ************************************************************************** //
77 // ************** delim_policy ************** //
78 // ************************************************************************** //
79
80 template<typename CharT,typename CharCompare>
81 class delim_policy {
82 typedef basic_cstring<CharT const> cstring;
83 public:
84 // Constructor
delim_policy(ti_delimeter_type t=dt_char,cstring d=cstring ())85 explicit delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() )
86 : m_type( t )
87 {
88 set_delimeters( d );
89 }
90
set_delimeters(ti_delimeter_type t)91 void set_delimeters( ti_delimeter_type t ) { m_type = t; }
92 template<typename Src>
set_delimeters(Src d)93 void set_delimeters( Src d )
94 {
95 nfp::optionally_assign( m_delimeters, d );
96
97 if( !m_delimeters.is_empty() )
98 m_type = dt_char;
99 }
100
operator ()(CharT c)101 bool operator()( CharT c )
102 {
103 switch( m_type ) {
104 case dt_char: {
105 BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
106 if( CharCompare()( delim, c ) )
107 return true;
108
109 return false;
110 }
111 case dt_ispunct:
112 return (std::ispunct)( c ) != 0;
113 case dt_isspace:
114 return (std::isspace)( c ) != 0;
115 case dt_none:
116 return false;
117 }
118
119 return false;
120 }
121
122 private:
123 // Data members
124 cstring m_delimeters;
125 ti_delimeter_type m_type;
126 };
127
128 // ************************************************************************** //
129 // ************** token_assigner ************** //
130 // ************************************************************************** //
131
132 template<typename TraversalTag>
133 struct token_assigner {
134 #if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
135 template<typename Iterator, typename C, typename T>
assignboost::unit_test::ut_detail::token_assigner136 static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
137 { for( ; b != e; ++b ) t += *b; }
138
139 template<typename Iterator, typename C>
assignboost::unit_test::ut_detail::token_assigner140 static void assign( Iterator b, Iterator e, basic_cstring<C>& t ) { t.assign( b, e ); }
141 #else
142 template<typename Iterator, typename Token>
143 static void assign( Iterator b, Iterator e, Token& t ) { t.assign( b, e ); }
144 #endif
145 template<typename Iterator, typename Token>
append_moveboost::unit_test::ut_detail::token_assigner146 static void append_move( Iterator& b, Token& ) { ++b; }
147 };
148
149 //____________________________________________________________________________//
150
151 template<>
152 struct token_assigner<single_pass_traversal_tag> {
153 template<typename Iterator, typename Token>
assignboost::unit_test::ut_detail::token_assigner154 static void assign( Iterator b, Iterator e, Token& t ) {}
155
156 template<typename Iterator, typename Token>
append_moveboost::unit_test::ut_detail::token_assigner157 static void append_move( Iterator& b, Token& t ) { t += *b; ++b; }
158 };
159
160 } // namespace ut_detail
161
162 // ************************************************************************** //
163 // ************** modifiers ************** //
164 // ************************************************************************** //
165
166 namespace {
167 nfp::keyword<struct dropped_delimeters_t > dropped_delimeters;
168 nfp::keyword<struct kept_delimeters_t > kept_delimeters;
169 nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
170 nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
171 }
172
173 // ************************************************************************** //
174 // ************** token_iterator_base ************** //
175 // ************************************************************************** //
176
177 template<typename Derived,
178 typename CharT,
179 typename CharCompare = ut_detail::default_char_compare<CharT>,
180 typename ValueType = basic_cstring<CharT const>,
181 typename Reference = basic_cstring<CharT const>,
182 typename Traversal = forward_traversal_tag>
183 class token_iterator_base
184 : public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
185 typedef basic_cstring<CharT const> cstring;
186 typedef ut_detail::delim_policy<CharT,CharCompare> delim_policy;
187 typedef input_iterator_facade<Derived,ValueType,Reference,Traversal> base;
188
189 protected:
190 // Constructor
token_iterator_base()191 explicit token_iterator_base()
192 : m_is_dropped( dt_isspace )
193 , m_is_kept( dt_ispunct )
194 , m_keep_empty_tokens( false )
195 , m_tokens_left( static_cast<std::size_t>(-1) )
196 , m_token_produced( false )
197 {
198 }
199
200 template<typename Modifier>
201 void
apply_modifier(Modifier const & m)202 apply_modifier( Modifier const& m )
203 {
204 if( m.has( dropped_delimeters ) )
205 m_is_dropped.set_delimeters( m[dropped_delimeters] );
206
207 if( m.has( kept_delimeters ) )
208 m_is_kept.set_delimeters( m[kept_delimeters] );
209
210 if( m.has( keep_empty_tokens ) )
211 m_keep_empty_tokens = true;
212
213 nfp::optionally_assign( m_tokens_left, m, max_tokens );
214 }
215
216 template<typename Iter>
get(Iter & begin,Iter end)217 bool get( Iter& begin, Iter end )
218 {
219 typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
220 Iter check_point;
221
222 this->m_value.clear();
223
224 if( !m_keep_empty_tokens ) {
225 while( begin != end && m_is_dropped( *begin ) )
226 ++begin;
227
228 if( begin == end )
229 return false;
230
231 check_point = begin;
232
233 if( m_tokens_left == 1 )
234 while( begin != end )
235 Assigner::append_move( begin, this->m_value );
236 else if( m_is_kept( *begin ) )
237 Assigner::append_move( begin, this->m_value );
238 else
239 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
240 Assigner::append_move( begin, this->m_value );
241
242 --m_tokens_left;
243 }
244 else { // m_keep_empty_tokens is true
245 check_point = begin;
246
247 if( begin == end ) {
248 if( m_token_produced )
249 return false;
250
251 m_token_produced = true;
252 }
253 if( m_is_kept( *begin ) ) {
254 if( m_token_produced )
255 Assigner::append_move( begin, this->m_value );
256
257 m_token_produced = !m_token_produced;
258 }
259 else if( !m_token_produced && m_is_dropped( *begin ) )
260 m_token_produced = true;
261 else {
262 if( m_is_dropped( *begin ) )
263 check_point = ++begin;
264
265 while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
266 Assigner::append_move( begin, this->m_value );
267
268 m_token_produced = true;
269 }
270 }
271
272 Assigner::assign( check_point, begin, this->m_value );
273
274 return true;
275 }
276
277 private:
278 // Data members
279 delim_policy m_is_dropped;
280 delim_policy m_is_kept;
281 bool m_keep_empty_tokens;
282 std::size_t m_tokens_left;
283 bool m_token_produced;
284 };
285
286 // ************************************************************************** //
287 // ************** basic_string_token_iterator ************** //
288 // ************************************************************************** //
289
290 template<typename CharT,
291 typename CharCompare = ut_detail::default_char_compare<CharT> >
292 class basic_string_token_iterator
293 : public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
294 typedef basic_cstring<CharT const> cstring;
295 typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
296 public:
basic_string_token_iterator()297 explicit basic_string_token_iterator() {}
basic_string_token_iterator(cstring src)298 explicit basic_string_token_iterator( cstring src )
299 : m_src( src )
300 {
301 this->init();
302 }
303
304 template<typename Src, typename Modifier>
basic_string_token_iterator(Src src,Modifier const & m)305 basic_string_token_iterator( Src src, Modifier const& m )
306 : m_src( src )
307 {
308 this->apply_modifier( m );
309
310 this->init();
311 }
312
313 private:
314 friend class input_iterator_core_access;
315
316 // input iterator implementation
get()317 bool get()
318 {
319 typename cstring::iterator begin = m_src.begin();
320 bool res = base::get( begin, m_src.end() );
321
322 m_src.assign( begin, m_src.end() );
323
324 return res;
325 }
326
327 // Data members
328 cstring m_src;
329 };
330
331 typedef basic_string_token_iterator<char> string_token_iterator;
332 typedef basic_string_token_iterator<wchar_t> wstring_token_iterator;
333
334 // ************************************************************************** //
335 // ************** range_token_iterator ************** //
336 // ************************************************************************** //
337
338 template<typename Iter,
339 typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
340 typename ValueType = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
341 typename Reference = ValueType const&>
342 class range_token_iterator
343 : public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
344 typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
345 typedef basic_cstring<typename ValueType::value_type> cstring;
346 typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
347 typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
348 public:
range_token_iterator()349 explicit range_token_iterator() {}
range_token_iterator(Iter begin,Iter end=Iter ())350 explicit range_token_iterator( Iter begin, Iter end = Iter() )
351 : m_begin( begin ), m_end( end )
352 {
353 this->init();
354 }
range_token_iterator(range_token_iterator const & rhs)355 range_token_iterator( range_token_iterator const& rhs )
356 : base( rhs )
357 {
358 if( this->m_valid ) {
359 m_begin = rhs.m_begin;
360 m_end = rhs.m_end;
361 }
362 }
363
364 template<typename Modifier>
range_token_iterator(Iter begin,Iter end,Modifier const & m)365 range_token_iterator( Iter begin, Iter end, Modifier const& m )
366 : m_begin( begin ), m_end( end )
367 {
368 this->apply_modifier( m );
369
370 this->init();
371 }
372
373 private:
374 friend class input_iterator_core_access;
375
376 // input iterator implementation
get()377 bool get()
378 {
379 return base::get( m_begin, m_end );
380 }
381
382 // Data members
383 Iter m_begin;
384 Iter m_end;
385 };
386
387 // ************************************************************************** //
388 // ************** make_range_token_iterator ************** //
389 // ************************************************************************** //
390
391 template<typename Iter>
392 inline range_token_iterator<Iter>
make_range_token_iterator(Iter begin,Iter end=Iter ())393 make_range_token_iterator( Iter begin, Iter end = Iter() )
394 {
395 return range_token_iterator<Iter>( begin, end );
396 }
397
398 //____________________________________________________________________________//
399
400 template<typename Iter,typename Modifier>
401 inline range_token_iterator<Iter>
make_range_token_iterator(Iter begin,Iter end,Modifier const & m)402 make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
403 {
404 return range_token_iterator<Iter>( begin, end, m );
405 }
406
407 //____________________________________________________________________________//
408
409 } // namespace unit_test
410
411 } // namespace boost
412
413 //____________________________________________________________________________//
414
415 #include <boost/test/detail/enable_warnings.hpp>
416
417 #endif // BOOST_TOKEN_ITERATOR_HPP_071894GER
418
419