1 /*=============================================================================
2 Boost.Wave: A Standard compliant C++ preprocessor library
3
4 Re2C based C++ lexer
5
6 http://www.boost.org/
7
8 Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
9 Software License, Version 1.0. (See accompanying file
10 LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
11 =============================================================================*/
12
13 #if !defined(BOOST_CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED)
14 #define BOOST_CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED
15
16 #include <boost/assert.hpp>
17
18 #include <boost/wave/wave_config.hpp>
19 #include <boost/wave/token_ids.hpp>
20 #include <boost/wave/cpplexer/cpplexer_exceptions.hpp>
21
22 // this must occur after all of the includes and before any code appears
23 #ifdef BOOST_HAS_ABI_HEADERS
24 #include BOOST_ABI_PREFIX
25 #endif
26
27 // suppress warnings about dependent classes not being exported from the dll
28 #ifdef BOOST_MSVC
29 #pragma warning(push)
30 #pragma warning(disable : 4251 4231 4660)
31 #endif
32
33 ///////////////////////////////////////////////////////////////////////////////
34
35 #define YYCTYPE uchar
36 #define YYCURSOR cursor
37 #define YYLIMIT limit
38 #define YYMARKER marker
39 #define YYFILL(n) \
40 { \
41 cursor = uchar_wrapper(fill(s, cursor), cursor.column); \
42 limit = uchar_wrapper (s->lim); \
43 } \
44 /**/
45
46 #include <iosfwd>
47
48 ///////////////////////////////////////////////////////////////////////////////
49 #define BOOST_WAVE_UPDATE_CURSOR() \
50 { \
51 s->line += count_backslash_newlines(s, cursor); \
52 s->curr_column = cursor.column; \
53 s->cur = cursor; \
54 s->lim = limit; \
55 s->ptr = marker; \
56 } \
57 /**/
58
59 ///////////////////////////////////////////////////////////////////////////////
60 #define BOOST_WAVE_RET(i) \
61 { \
62 BOOST_WAVE_UPDATE_CURSOR() \
63 if (s->cur > s->lim) \
64 return T_EOF; /* may happen for empty files */ \
65 return (i); \
66 } \
67 /**/
68
69 ///////////////////////////////////////////////////////////////////////////////
70
71 namespace boost {
72 namespace wave {
73 namespace cpplexer {
74 namespace re2clex {
75
76 template<typename Iterator>
77 struct Scanner;
78
79 ///////////////////////////////////////////////////////////////////////////////
80 // The scanner function to call whenever a new token is requested
81 template<typename Iterator>
82 BOOST_WAVE_DECL boost::wave::token_id scan(Scanner<Iterator> *s);
83 ///////////////////////////////////////////////////////////////////////////////
84
85 ///////////////////////////////////////////////////////////////////////////////
86 // Utility functions
87
88 #define RE2C_ASSERT BOOST_ASSERT
89
90 template<typename Iterator>
get_one_char(Scanner<Iterator> * s)91 int get_one_char(Scanner<Iterator> *s)
92 {
93 RE2C_ASSERT(s->first <= s->act && s->act <= s->last);
94 if (s->act < s->last)
95 return *(s->act)++;
96 return -1;
97 }
98
99 template<typename Iterator>
rewind_stream(Scanner<Iterator> * s,int cnt)100 std::ptrdiff_t rewind_stream (Scanner<Iterator> *s, int cnt)
101 {
102 std::advance(s->act, cnt);
103 RE2C_ASSERT(s->first <= s->act && s->act <= s->last);
104 return std::distance(s->first, s->act);
105 }
106
107 template<typename Iterator>
get_first_eol_offset(Scanner<Iterator> * s)108 std::size_t get_first_eol_offset(Scanner<Iterator>* s)
109 {
110 if (!AQ_EMPTY(s->eol_offsets))
111 {
112 return s->eol_offsets->queue[s->eol_offsets->head];
113 }
114 else
115 {
116 return (unsigned int)-1;
117 }
118 }
119
120 template<typename Iterator>
adjust_eol_offsets(Scanner<Iterator> * s,std::size_t adjustment)121 void adjust_eol_offsets(Scanner<Iterator>* s, std::size_t adjustment)
122 {
123 aq_queue q;
124 std::size_t i;
125
126 if (!s->eol_offsets)
127 s->eol_offsets = aq_create();
128
129 q = s->eol_offsets;
130
131 if (AQ_EMPTY(q))
132 return;
133
134 i = q->head;
135 while (i != q->tail)
136 {
137 if (adjustment > q->queue[i])
138 q->queue[i] = 0;
139 else
140 q->queue[i] -= adjustment;
141 ++i;
142 if (i == q->max_size)
143 i = 0;
144 }
145 if (adjustment > q->queue[i])
146 q->queue[i] = 0;
147 else
148 q->queue[i] -= adjustment;
149 }
150
151 template<typename Iterator>
count_backslash_newlines(Scanner<Iterator> * s,uchar * cursor)152 int count_backslash_newlines(Scanner<Iterator> *s, uchar *cursor)
153 {
154 std::size_t diff, offset;
155 int skipped = 0;
156
157 /* figure out how many backslash-newlines skipped over unknowingly. */
158 diff = cursor - s->bot;
159 offset = get_first_eol_offset(s);
160 while (offset <= diff && offset != (unsigned int)-1)
161 {
162 skipped++;
163 aq_pop(s->eol_offsets);
164 offset = get_first_eol_offset(s);
165 }
166 return skipped;
167 }
168
169 BOOST_WAVE_DECL bool is_backslash(uchar *p, uchar *end, int &len);
170
171 #define BOOST_WAVE_BSIZE 196608
172 template<typename Iterator>
fill(Scanner<Iterator> * s,uchar * cursor)173 uchar *fill(Scanner<Iterator> *s, uchar *cursor)
174 {
175 using namespace std; // some systems have memcpy etc. in namespace std
176 if(!s->eof)
177 {
178 uchar* p;
179 std::ptrdiff_t cnt = s->tok - s->bot;
180 if(cnt)
181 {
182 if (NULL == s->lim)
183 s->lim = s->top;
184 memmove(s->bot, s->tok, s->lim - s->tok);
185 s->tok = s->cur = s->bot;
186 s->ptr -= cnt;
187 cursor -= cnt;
188 s->lim -= cnt;
189 adjust_eol_offsets(s, cnt);
190 }
191
192 if((s->top - s->lim) < BOOST_WAVE_BSIZE)
193 {
194 uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BOOST_WAVE_BSIZE)*sizeof(uchar));
195 if (buf == 0)
196 {
197 (*s->error_proc)(s, lexing_exception::unexpected_error,
198 "Out of memory!");
199
200 /* get the scanner to stop */
201 *cursor = 0;
202 return cursor;
203 }
204
205 memmove(buf, s->tok, s->lim - s->tok);
206 s->tok = s->cur = buf;
207 s->ptr = &buf[s->ptr - s->bot];
208 cursor = &buf[cursor - s->bot];
209 s->lim = &buf[s->lim - s->bot];
210 s->top = &s->lim[BOOST_WAVE_BSIZE];
211 free(s->bot);
212 s->bot = buf;
213 }
214
215 cnt = std::distance(s->act, s->last);
216 if (cnt > BOOST_WAVE_BSIZE)
217 cnt = BOOST_WAVE_BSIZE;
218 uchar * dst = s->lim;
219 for (std::ptrdiff_t idx = 0; idx < cnt; ++idx)
220 {
221 *dst++ = *s->act++;
222 }
223
224 if (cnt != BOOST_WAVE_BSIZE)
225 {
226 s->eof = &s->lim[cnt]; *(s->eof)++ = '\0';
227 }
228
229 /* backslash-newline erasing time */
230
231 /* first scan for backslash-newline and erase them */
232 for (p = s->lim; p < s->lim + cnt - 2; ++p)
233 {
234 int len = 0;
235 if (is_backslash(p, s->lim + cnt, len))
236 {
237 if (*(p+len) == '\n')
238 {
239 int offset = len + 1;
240 memmove(p, p + offset, s->lim + cnt - p - offset);
241 cnt -= offset;
242 --p;
243 aq_enqueue(s->eol_offsets, p - s->bot + 1);
244 }
245 else if (*(p+len) == '\r')
246 {
247 if (*(p+len+1) == '\n')
248 {
249 int offset = len + 2;
250 memmove(p, p + offset, s->lim + cnt - p - offset);
251 cnt -= offset;
252 --p;
253 }
254 else
255 {
256 int offset = len + 1;
257 memmove(p, p + offset, s->lim + cnt - p - offset);
258 cnt -= offset;
259 --p;
260 }
261 aq_enqueue(s->eol_offsets, p - s->bot + 1);
262 }
263 }
264 }
265
266 /* FIXME: the following code should be fixed to recognize correctly the
267 trigraph backslash token */
268
269 /* check to see if what we just read ends in a backslash */
270 if (cnt >= 2)
271 {
272 uchar last = s->lim[cnt-1];
273 uchar last2 = s->lim[cnt-2];
274 /* check \ EOB */
275 if (last == '\\')
276 {
277 int next = get_one_char(s);
278 /* check for \ \n or \ \r or \ \r \n straddling the border */
279 if (next == '\n')
280 {
281 --cnt; /* chop the final \, we've already read the \n. */
282 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
283 }
284 else if (next == '\r')
285 {
286 int next2 = get_one_char(s);
287 if (next2 == '\n')
288 {
289 --cnt; /* skip the backslash */
290 }
291 else
292 {
293 /* rewind one, and skip one char */
294 rewind_stream(s, -1);
295 --cnt;
296 }
297 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
298 }
299 else if (next != -1) /* -1 means end of file */
300 {
301 /* next was something else, so rewind the stream */
302 rewind_stream(s, -1);
303 }
304 }
305 /* check \ \r EOB */
306 else if (last == '\r' && last2 == '\\')
307 {
308 int next = get_one_char(s);
309 if (next == '\n')
310 {
311 cnt -= 2; /* skip the \ \r */
312 }
313 else
314 {
315 /* rewind one, and skip two chars */
316 rewind_stream(s, -1);
317 cnt -= 2;
318 }
319 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
320 }
321 /* check \ \n EOB */
322 else if (last == '\n' && last2 == '\\')
323 {
324 cnt -= 2;
325 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
326 }
327 }
328
329 s->lim += cnt;
330 if (s->eof) /* eof needs adjusting if we erased backslash-newlines */
331 {
332 s->eof = s->lim;
333 *(s->eof)++ = '\0';
334 }
335 }
336 return cursor;
337 }
338 #undef BOOST_WAVE_BSIZE
339
340 ///////////////////////////////////////////////////////////////////////////////
341 // Special wrapper class holding the current cursor position
342 struct BOOST_WAVE_DECL uchar_wrapper
343 {
344 uchar_wrapper (uchar *base_cursor, std::size_t column = 1);
345
346 uchar_wrapper& operator++();
347
348 uchar_wrapper& operator--();
349
350 uchar operator* () const;
351
352 operator uchar *() const;
353
354 friend BOOST_WAVE_DECL std::ptrdiff_t
355 operator- (uchar_wrapper const& lhs, uchar_wrapper const& rhs);
356
357 uchar *base_cursor;
358 std::size_t column;
359 };
360
361
362 ///////////////////////////////////////////////////////////////////////////////
363 template<typename Iterator>
scan(Scanner<Iterator> * s)364 boost::wave::token_id scan(Scanner<Iterator> *s)
365 {
366 BOOST_ASSERT(0 != s->error_proc); // error handler must be given
367
368 uchar_wrapper cursor (s->tok = s->cur, s->column = s->curr_column);
369 uchar_wrapper marker (s->ptr);
370 uchar_wrapper limit (s->lim);
371
372 typedef BOOST_WAVE_STRINGTYPE string_type;
373 string_type rawstringdelim; // for use with C++11 raw string literals
374
375 // include the correct Re2C token definition rules
376 #if (defined (__FreeBSD__) || defined (__DragonFly__) || defined (__OpenBSD__)) && defined (T_DIVIDE)
377 #undef T_DIVIDE
378 #endif
379 #if BOOST_WAVE_USE_STRICT_LEXER != 0
380 #include "strict_cpp_re.inc"
381 #else
382 #include "cpp_re.inc"
383 #endif
384
385 } /* end of scan */
386
387 ///////////////////////////////////////////////////////////////////////////////
388
389 } // namespace re2clex
390 } // namespace cpplexer
391 } // namespace wave
392 } // namespace boost
393
394 #ifdef BOOST_MSVC
395 #pragma warning(pop)
396 #endif
397
398 #undef BOOST_WAVE_RET
399 #undef YYCTYPE
400 #undef YYCURSOR
401 #undef YYLIMIT
402 #undef YYMARKER
403 #undef YYFILL
404
405 // the suffix header occurs after all of the code
406 #ifdef BOOST_HAS_ABI_HEADERS
407 #include BOOST_ABI_SUFFIX
408 #endif
409
410 #endif // !defined(BOOST_CPP_RE_HPP_B76C4F5E_63E9_4B8A_9975_EC32FA6BF027_INCLUDED)
411