1 /*=============================================================================
2 Boost.Wave: A Standard compliant C++ preprocessor library
3
4 A generic C++ lexer token definition
5
6 http://www.boost.org/
7
8 Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
9 Software License, Version 1.0. (See accompanying file
10 LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
11 =============================================================================*/
12
13 #if !defined(BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
14 #define BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED
15
16 #include <boost/wave/wave_config.hpp>
17 #if BOOST_WAVE_SERIALIZATION != 0
18 #include <boost/serialization/serialization.hpp>
19 #endif
20 #include <boost/wave/util/file_position.hpp>
21 #include <boost/wave/token_ids.hpp>
22 #include <boost/wave/language_support.hpp>
23
24 #include <boost/throw_exception.hpp>
25 #include <boost/pool/singleton_pool.hpp>
26 #include <boost/detail/atomic_count.hpp>
27
28 // this must occur after all of the includes and before any code appears
29 #ifdef BOOST_HAS_ABI_HEADERS
30 #include BOOST_ABI_PREFIX
31 #endif
32
33 ///////////////////////////////////////////////////////////////////////////////
34 namespace boost {
35 namespace wave {
36 namespace cpplexer {
37
38 namespace impl {
39
40 template <typename StringTypeT, typename PositionT>
41 class token_data
42 {
43 public:
44 typedef StringTypeT string_type;
45 typedef PositionT position_type;
46
47 // default constructed tokens correspond to EOI tokens
token_data()48 token_data()
49 : id(T_EOI), refcnt(1)
50 {}
51
52 // construct an invalid token
token_data(int)53 explicit token_data(int)
54 : id(T_UNKNOWN), refcnt(1)
55 {}
56
token_data(token_id id_,string_type const & value_,position_type const & pos_)57 token_data(token_id id_, string_type const &value_, position_type const &pos_)
58 : id(id_), value(value_), pos(pos_), refcnt(1)
59 {}
60
token_data(token_data const & rhs)61 token_data(token_data const& rhs)
62 : id(rhs.id), value(rhs.value), pos(rhs.pos), refcnt(1)
63 {}
64
~token_data()65 ~token_data()
66 {}
67
addref()68 std::size_t addref() { return ++refcnt; }
release()69 std::size_t release() { return --refcnt; }
get_refcnt() const70 std::size_t get_refcnt() const { return refcnt; }
71
72 // accessors
operator token_id() const73 operator token_id() const { return id; }
get_value() const74 string_type const &get_value() const { return value; }
get_position() const75 position_type const &get_position() const { return pos; }
76
set_token_id(token_id id_)77 void set_token_id (token_id id_) { id = id_; }
set_value(string_type const & value_)78 void set_value (string_type const &value_) { value = value_; }
set_position(position_type const & pos_)79 void set_position (position_type const &pos_) { pos = pos_; }
80
operator ==(token_data const & lhs,token_data const & rhs)81 friend bool operator== (token_data const& lhs, token_data const& rhs)
82 {
83 // two tokens are considered equal even if they refer to different
84 // positions
85 return (lhs.id == rhs.id && lhs.value == rhs.value) ? true : false;
86 }
87
init(token_id id_,string_type const & value_,position_type const & pos_)88 void init(token_id id_, string_type const &value_, position_type const &pos_)
89 {
90 BOOST_ASSERT(refcnt == 1);
91 id = id_;
92 value = value_;
93 pos = pos_;
94 }
95
init(token_data const & rhs)96 void init(token_data const& rhs)
97 {
98 BOOST_ASSERT(refcnt == 1);
99 id = rhs.id;
100 value = rhs.value;
101 pos = rhs.pos;
102 }
103
104 static void *operator new(std::size_t size);
105 static void operator delete(void *p, std::size_t size);
106
107 #if defined(BOOST_SPIRIT_DEBUG)
108 // debug support
print(std::ostream & stream) const109 void print (std::ostream &stream) const
110 {
111 stream << get_token_name(id) << "(";
112 for (std::size_t i = 0; i < value.size(); ++i) {
113 switch (value[i]) {
114 case '\r': stream << "\\r"; break;
115 case '\n': stream << "\\n"; break;
116 default:
117 stream << value[i];
118 break;
119 }
120 }
121 stream << ")";
122 }
123 #endif // defined(BOOST_SPIRIT_DEBUG)
124
125 #if BOOST_WAVE_SERIALIZATION != 0
126 friend class boost::serialization::access;
127 template<typename Archive>
serialize(Archive & ar,const unsigned int version)128 void serialize(Archive &ar, const unsigned int version)
129 {
130 using namespace boost::serialization;
131 ar & make_nvp("id", id);
132 ar & make_nvp("value", value);
133 ar & make_nvp("position", pos);
134 }
135 #endif
136
137 private:
138 token_id id; // the token id
139 string_type value; // the text, which was parsed into this token
140 position_type pos; // the original file position
141 boost::detail::atomic_count refcnt;
142 };
143
144 ///////////////////////////////////////////////////////////////////////////////
145 struct token_data_tag {};
146
147 template <typename StringTypeT, typename PositionT>
148 inline void *
operator new(std::size_t size)149 token_data<StringTypeT, PositionT>::operator new(std::size_t size)
150 {
151 BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
152 typedef boost::singleton_pool<
153 token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
154 > pool_type;
155
156 void *ret = pool_type::malloc();
157 if (0 == ret)
158 boost::throw_exception(std::bad_alloc());
159 return ret;
160 }
161
162 template <typename StringTypeT, typename PositionT>
163 inline void
operator delete(void * p,std::size_t size)164 token_data<StringTypeT, PositionT>::operator delete(void *p, std::size_t size)
165 {
166 BOOST_ASSERT(sizeof(token_data<StringTypeT, PositionT>) == size);
167 typedef boost::singleton_pool<
168 token_data_tag, sizeof(token_data<StringTypeT, PositionT>)
169 > pool_type;
170
171 if (0 != p)
172 pool_type::free(p);
173 }
174
175 } // namespace impl
176
177 ///////////////////////////////////////////////////////////////////////////////
178 // forward declaration of the token type
179 template <typename PositionT = boost::wave::util::file_position_type>
180 class lex_token;
181
182 ///////////////////////////////////////////////////////////////////////////////
183 //
184 // lex_token
185 //
186 ///////////////////////////////////////////////////////////////////////////////
187
188 template <typename PositionT>
189 class lex_token
190 {
191 public:
192 typedef BOOST_WAVE_STRINGTYPE string_type;
193 typedef PositionT position_type;
194
195 private:
196 typedef impl::token_data<string_type, position_type> data_type;
197
198 public:
199 // default constructed tokens correspond to EOI tokens
lex_token()200 lex_token()
201 : data(0)
202 {}
203
204 // construct an invalid token
lex_token(int)205 explicit lex_token(int)
206 : data(new data_type(0))
207 {}
208
lex_token(lex_token const & rhs)209 lex_token(lex_token const& rhs)
210 : data(rhs.data)
211 {
212 if (0 != data)
213 data->addref();
214 }
215
lex_token(token_id id_,string_type const & value_,PositionT const & pos_)216 lex_token(token_id id_, string_type const &value_, PositionT const &pos_)
217 : data(new data_type(id_, value_, pos_))
218 {}
219
~lex_token()220 ~lex_token()
221 {
222 if (0 != data && 0 == data->release())
223 delete data;
224 data = 0;
225 }
226
operator =(lex_token const & rhs)227 lex_token& operator=(lex_token const& rhs)
228 {
229 if (&rhs != this) {
230 if (0 != data && 0 == data->release())
231 delete data;
232
233 data = rhs.data;
234 if (0 != data)
235 data->addref();
236 }
237 return *this;
238 }
239
240 // accessors
operator token_id() const241 operator token_id() const { return 0 != data ? token_id(*data) : T_EOI; }
get_value() const242 string_type const &get_value() const { return data->get_value(); }
get_position() const243 position_type const &get_position() const { return data->get_position(); }
is_eoi() const244 bool is_eoi() const { return 0 == data || token_id(*data) == T_EOI; }
is_valid() const245 bool is_valid() const { return 0 != data && token_id(*data) != T_UNKNOWN; }
246
set_token_id(token_id id_)247 void set_token_id (token_id id_) { make_unique(); data->set_token_id(id_); }
set_value(string_type const & value_)248 void set_value (string_type const &value_) { make_unique(); data->set_value(value_); }
set_position(position_type const & pos_)249 void set_position (position_type const &pos_) { make_unique(); data->set_position(pos_); }
250
operator ==(lex_token const & lhs,lex_token const & rhs)251 friend bool operator== (lex_token const& lhs, lex_token const& rhs)
252 {
253 if (0 == rhs.data)
254 return 0 == lhs.data;
255 if (0 == lhs.data)
256 return false;
257 return *(lhs.data) == *(rhs.data);
258 }
259
260 // debug support
261 #if BOOST_WAVE_DUMP_PARSE_TREE != 0
262 // access functions for the tree_to_xml functionality
get_token_id(lex_token const & t)263 static int get_token_id(lex_token const &t)
264 { return token_id(t); }
get_token_value(lex_token const & t)265 static string_type get_token_value(lex_token const &t)
266 { return t.get_value(); }
267 #endif
268
269 #if defined(BOOST_SPIRIT_DEBUG)
270 // debug support
print(std::ostream & stream) const271 void print (std::ostream &stream) const
272 {
273 data->print(stream);
274 }
275 #endif // defined(BOOST_SPIRIT_DEBUG)
276
277 private:
278 #if BOOST_WAVE_SERIALIZATION != 0
279 friend class boost::serialization::access;
280 template<typename Archive>
serialize(Archive & ar,const unsigned int version)281 void serialize(Archive &ar, const unsigned int version)
282 {
283 data->serialize(ar, version);
284 }
285 #endif
286
287 // make a unique copy of the current object
make_unique()288 void make_unique()
289 {
290 if (1 == data->get_refcnt())
291 return;
292
293 data_type* newdata = new data_type(*data) ;
294
295 data->release(); // release this reference, can't get zero
296 data = newdata;
297 }
298
299 data_type* data;
300 };
301
302 ///////////////////////////////////////////////////////////////////////////////
303 // This overload is needed by the multi_pass/functor_input_policy to
304 // validate a token instance. It has to be defined in the same namespace
305 // as the token class itself to allow ADL to find it.
306 ///////////////////////////////////////////////////////////////////////////////
307 template <typename Position>
308 inline bool
token_is_valid(lex_token<Position> const & t)309 token_is_valid(lex_token<Position> const& t)
310 {
311 return t.is_valid();
312 }
313
314 ///////////////////////////////////////////////////////////////////////////////
315 #if defined(BOOST_SPIRIT_DEBUG)
316 template <typename PositionT>
317 inline std::ostream &
operator <<(std::ostream & stream,lex_token<PositionT> const & object)318 operator<< (std::ostream &stream, lex_token<PositionT> const &object)
319 {
320 object.print(stream);
321 return stream;
322 }
323 #endif // defined(BOOST_SPIRIT_DEBUG)
324
325 ///////////////////////////////////////////////////////////////////////////////
326 } // namespace cpplexer
327 } // namespace wave
328 } // namespace boost
329
330 // the suffix header occurs after all of the code
331 #ifdef BOOST_HAS_ABI_HEADERS
332 #include BOOST_ABI_SUFFIX
333 #endif
334
335 #endif // !defined(BOOST_CPP_TOKEN_HPP_53A13BD2_FBAA_444B_9B8B_FCB225C2BBA8_INCLUDED)
336