• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2 ** $Id: llex.c,v 2.96.1.1 2017/04/19 17:20:42 roberto Exp $
3 ** Lexical Analyzer
4 ** See Copyright Notice in lua.h
5 */
6 
7 #define llex_c
8 #define LUA_CORE
9 
10 #include "lprefix.h"
11 
12 
13 #include <locale.h>
14 #include <string.h>
15 
16 #include "lua.h"
17 
18 #include "lctype.h"
19 #include "ldebug.h"
20 #include "ldo.h"
21 #include "lgc.h"
22 #include "llex.h"
23 #include "lobject.h"
24 #include "lparser.h"
25 #include "lstate.h"
26 #include "lstring.h"
27 #include "ltable.h"
28 #include "lzio.h"
29 
30 
31 
32 #define next(ls) (ls->current = zgetc(ls->z))
33 
34 
35 
36 #define currIsNewline(ls)	(ls->current == '\n' || ls->current == '\r')
37 
38 
39 /* ORDER RESERVED */
40 static const char *const luaX_tokens [] = {
41     "and", "break", "do", "else", "elseif",
42     "end", "false", "for", "function", "goto", "if",
43     "in", "local", "nil", "not", "or", "repeat",
44     "return", "then", "true", "until", "while",
45     "//", "..", "...", "==", ">=", "<=", "~=",
46     "<<", ">>", "::", "<eof>",
47     "<number>", "<integer>", "<name>", "<string>"
48 };
49 
50 
51 #define save_and_next(ls) (save(ls, ls->current), next(ls))
52 
53 
54 static l_noret lexerror (LexState *ls, const char *msg, int token);
55 
56 
save(LexState * ls,int c)57 static void save (LexState *ls, int c) {
58   Mbuffer *b = ls->buff;
59   if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) {
60     size_t newsize;
61     if (luaZ_sizebuffer(b) >= MAX_SIZE/2)
62       lexerror(ls, "lexical element too long", 0);
63     newsize = luaZ_sizebuffer(b) * 2;
64     luaZ_resizebuffer(ls->L, b, newsize);
65   }
66   b->buffer[luaZ_bufflen(b)++] = cast(char, c);
67 }
68 
69 
luaX_init(lua_State * L)70 void luaX_init (lua_State *L) {
71   int i;
72   TString *e = luaS_newliteral(L, LUA_ENV);  /* create env name */
73   luaC_fix(L, obj2gco(e));  /* never collect this name */
74   for (i=0; i<NUM_RESERVED; i++) {
75     TString *ts = luaS_new(L, luaX_tokens[i]);
76     luaC_fix(L, obj2gco(ts));  /* reserved words are never collected */
77     ts->extra = cast_byte(i+1);  /* reserved word */
78   }
79 }
80 
81 
luaX_token2str(LexState * ls,int token)82 const char *luaX_token2str (LexState *ls, int token) {
83   if (token < FIRST_RESERVED) {  /* single-byte symbols? */
84     lua_assert(token == cast_uchar(token));
85     return luaO_pushfstring(ls->L, "'%c'", token);
86   }
87   else {
88     const char *s = luaX_tokens[token - FIRST_RESERVED];
89     if (token < TK_EOS)  /* fixed format (symbols and reserved words)? */
90       return luaO_pushfstring(ls->L, "'%s'", s);
91     else  /* names, strings, and numerals */
92       return s;
93   }
94 }
95 
96 
txtToken(LexState * ls,int token)97 static const char *txtToken (LexState *ls, int token) {
98   switch (token) {
99     case TK_NAME: case TK_STRING:
100     case TK_FLT: case TK_INT:
101       save(ls, '\0');
102       return luaO_pushfstring(ls->L, "'%s'", luaZ_buffer(ls->buff));
103     default:
104       return luaX_token2str(ls, token);
105   }
106 }
107 
108 
lexerror(LexState * ls,const char * msg,int token)109 static l_noret lexerror (LexState *ls, const char *msg, int token) {
110   msg = luaG_addinfo(ls->L, msg, ls->source, ls->linenumber);
111   if (token)
112     luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token));
113   luaD_throw(ls->L, LUA_ERRSYNTAX);
114 }
115 
116 
luaX_syntaxerror(LexState * ls,const char * msg)117 l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
118   lexerror(ls, msg, ls->t.token);
119 }
120 
121 
122 /*
123 ** creates a new string and anchors it in scanner's table so that
124 ** it will not be collected until the end of the compilation
125 ** (by that time it should be anchored somewhere)
126 */
luaX_newstring(LexState * ls,const char * str,size_t l)127 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
128   lua_State *L = ls->L;
129   TValue *o;  /* entry for 'str' */
130   TString *ts = luaS_newlstr(L, str, l);  /* create new string */
131   setsvalue2s(L, L->top++, ts);  /* temporarily anchor it in stack */
132   o = luaH_set(L, ls->h, L->top - 1);
133   if (ttisnil(o)) {  /* not in use yet? */
134     /* boolean value does not need GC barrier;
135        table has no metatable, so it does not need to invalidate cache */
136     setbvalue(o, 1);  /* t[string] = true */
137     luaC_checkGC(L);
138   }
139   else {  /* string already present */
140     ts = tsvalue(keyfromval(o));  /* re-use value previously stored */
141   }
142   L->top--;  /* remove string from stack */
143   return ts;
144 }
145 
146 
147 /*
148 ** increment line number and skips newline sequence (any of
149 ** \n, \r, \n\r, or \r\n)
150 */
inclinenumber(LexState * ls)151 static void inclinenumber (LexState *ls) {
152   int old = ls->current;
153   lua_assert(currIsNewline(ls));
154   next(ls);  /* skip '\n' or '\r' */
155   if (currIsNewline(ls) && ls->current != old)
156     next(ls);  /* skip '\n\r' or '\r\n' */
157   if (++ls->linenumber >= MAX_INT)
158     lexerror(ls, "chunk has too many lines", 0);
159 }
160 
161 
luaX_setinput(lua_State * L,LexState * ls,ZIO * z,TString * source,int firstchar)162 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source,
163                     int firstchar) {
164   ls->t.token = 0;
165   ls->L = L;
166   ls->current = firstchar;
167   ls->lookahead.token = TK_EOS;  /* no look-ahead token */
168   ls->z = z;
169   ls->fs = NULL;
170   ls->linenumber = 1;
171   ls->lastline = 1;
172   ls->source = source;
173   ls->envn = luaS_newliteral(L, LUA_ENV);  /* get env name */
174   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
175 }
176 
177 
178 
179 /*
180 ** =======================================================
181 ** LEXICAL ANALYZER
182 ** =======================================================
183 */
184 
185 
check_next1(LexState * ls,int c)186 static int check_next1 (LexState *ls, int c) {
187   if (ls->current == c) {
188     next(ls);
189     return 1;
190   }
191   else return 0;
192 }
193 
194 
195 /*
196 ** Check whether current char is in set 'set' (with two chars) and
197 ** saves it
198 */
check_next2(LexState * ls,const char * set)199 static int check_next2 (LexState *ls, const char *set) {
200   lua_assert(set[2] == '\0');
201   if (ls->current == set[0] || ls->current == set[1]) {
202     save_and_next(ls);
203     return 1;
204   }
205   else return 0;
206 }
207 
208 
209 /* LUA_NUMBER */
210 /*
211 ** this function is quite liberal in what it accepts, as 'luaO_str2num'
212 ** will reject ill-formed numerals.
213 */
read_numeral(LexState * ls,SemInfo * seminfo)214 static int read_numeral (LexState *ls, SemInfo *seminfo) {
215   TValue obj;
216   const char *expo = "Ee";
217   int first = ls->current;
218   lua_assert(lisdigit(ls->current));
219   save_and_next(ls);
220   if (first == '0' && check_next2(ls, "xX"))  /* hexadecimal? */
221     expo = "Pp";
222   for (;;) {
223     if (check_next2(ls, expo))  /* exponent part? */
224       check_next2(ls, "-+");  /* optional exponent sign */
225     if (lisxdigit(ls->current))
226       save_and_next(ls);
227     else if (ls->current == '.')
228       save_and_next(ls);
229     else break;
230   }
231   save(ls, '\0');
232   if (luaO_str2num(luaZ_buffer(ls->buff), &obj) == 0)  /* format error? */
233     lexerror(ls, "malformed number", TK_FLT);
234   if (ttisinteger(&obj)) {
235     seminfo->i = ivalue(&obj);
236     return TK_INT;
237   }
238   else {
239     lua_assert(ttisfloat(&obj));
240     seminfo->r = fltvalue(&obj);
241     return TK_FLT;
242   }
243 }
244 
245 
246 /*
247 ** skip a sequence '[=*[' or ']=*]'; if sequence is well formed, return
248 ** its number of '='s; otherwise, return a negative number (-1 iff there
249 ** are no '='s after initial bracket)
250 */
skip_sep(LexState * ls)251 static int skip_sep (LexState *ls) {
252   int count = 0;
253   int s = ls->current;
254   lua_assert(s == '[' || s == ']');
255   save_and_next(ls);
256   while (ls->current == '=') {
257     save_and_next(ls);
258     count++;
259   }
260   return (ls->current == s) ? count : (-count) - 1;
261 }
262 
263 
read_long_string(LexState * ls,SemInfo * seminfo,int sep)264 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) {
265   int line = ls->linenumber;  /* initial line (for error message) */
266   save_and_next(ls);  /* skip 2nd '[' */
267   if (currIsNewline(ls))  /* string starts with a newline? */
268     inclinenumber(ls);  /* skip it */
269   for (;;) {
270     switch (ls->current) {
271       case EOZ: {  /* error */
272         const char *what = (seminfo ? "string" : "comment");
273         const char *msg = luaO_pushfstring(ls->L,
274                      "unfinished long %s (starting at line %d)", what, line);
275         lexerror(ls, msg, TK_EOS);
276         break;  /* to avoid warnings */
277       }
278       case ']': {
279         if (skip_sep(ls) == sep) {
280           save_and_next(ls);  /* skip 2nd ']' */
281           goto endloop;
282         }
283         break;
284       }
285       case '\n': case '\r': {
286         save(ls, '\n');
287         inclinenumber(ls);
288         if (!seminfo) luaZ_resetbuffer(ls->buff);  /* avoid wasting space */
289         break;
290       }
291       default: {
292         if (seminfo) save_and_next(ls);
293         else next(ls);
294       }
295     }
296   } endloop:
297   if (seminfo)
298     seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep),
299                                      luaZ_bufflen(ls->buff) - 2*(2 + sep));
300 }
301 
302 
esccheck(LexState * ls,int c,const char * msg)303 static void esccheck (LexState *ls, int c, const char *msg) {
304   if (!c) {
305     if (ls->current != EOZ)
306       save_and_next(ls);  /* add current to buffer for error message */
307     lexerror(ls, msg, TK_STRING);
308   }
309 }
310 
311 
gethexa(LexState * ls)312 static int gethexa (LexState *ls) {
313   save_and_next(ls);
314   esccheck (ls, lisxdigit(ls->current), "hexadecimal digit expected");
315   return luaO_hexavalue(ls->current);
316 }
317 
318 
readhexaesc(LexState * ls)319 static int readhexaesc (LexState *ls) {
320   int r = gethexa(ls);
321   r = (r << 4) + gethexa(ls);
322   luaZ_buffremove(ls->buff, 2);  /* remove saved chars from buffer */
323   return r;
324 }
325 
326 
readutf8esc(LexState * ls)327 static unsigned long readutf8esc (LexState *ls) {
328   unsigned long r;
329   int i = 4;  /* chars to be removed: '\', 'u', '{', and first digit */
330   save_and_next(ls);  /* skip 'u' */
331   esccheck(ls, ls->current == '{', "missing '{'");
332   r = gethexa(ls);  /* must have at least one digit */
333   while ((save_and_next(ls), lisxdigit(ls->current))) {
334     i++;
335     r = (r << 4) + luaO_hexavalue(ls->current);
336     esccheck(ls, r <= 0x10FFFF, "UTF-8 value too large");
337   }
338   esccheck(ls, ls->current == '}', "missing '}'");
339   next(ls);  /* skip '}' */
340   luaZ_buffremove(ls->buff, i);  /* remove saved chars from buffer */
341   return r;
342 }
343 
344 
utf8esc(LexState * ls)345 static void utf8esc (LexState *ls) {
346   char buff[UTF8BUFFSZ];
347   int n = luaO_utf8esc(buff, readutf8esc(ls));
348   for (; n > 0; n--)  /* add 'buff' to string */
349     save(ls, buff[UTF8BUFFSZ - n]);
350 }
351 
352 
readdecesc(LexState * ls)353 static int readdecesc (LexState *ls) {
354   int i;
355   int r = 0;  /* result accumulator */
356   for (i = 0; i < 3 && lisdigit(ls->current); i++) {  /* read up to 3 digits */
357     r = 10*r + ls->current - '0';
358     save_and_next(ls);
359   }
360   esccheck(ls, r <= UCHAR_MAX, "decimal escape too large");
361   luaZ_buffremove(ls->buff, i);  /* remove read digits from buffer */
362   return r;
363 }
364 
365 
read_string(LexState * ls,int del,SemInfo * seminfo)366 static void read_string (LexState *ls, int del, SemInfo *seminfo) {
367   save_and_next(ls);  /* keep delimiter (for error messages) */
368   while (ls->current != del) {
369     switch (ls->current) {
370       case EOZ:
371         lexerror(ls, "unfinished string", TK_EOS);
372         break;  /* to avoid warnings */
373       case '\n':
374       case '\r':
375         lexerror(ls, "unfinished string", TK_STRING);
376         break;  /* to avoid warnings */
377       case '\\': {  /* escape sequences */
378         int c;  /* final character to be saved */
379         save_and_next(ls);  /* keep '\\' for error messages */
380         switch (ls->current) {
381           case 'a': c = '\a'; goto read_save;
382           case 'b': c = '\b'; goto read_save;
383           case 'f': c = '\f'; goto read_save;
384           case 'n': c = '\n'; goto read_save;
385           case 'r': c = '\r'; goto read_save;
386           case 't': c = '\t'; goto read_save;
387           case 'v': c = '\v'; goto read_save;
388           case 'x': c = readhexaesc(ls); goto read_save;
389           case 'u': utf8esc(ls);  goto no_save;
390           case '\n': case '\r':
391             inclinenumber(ls); c = '\n'; goto only_save;
392           case '\\': case '\"': case '\'':
393             c = ls->current; goto read_save;
394           case EOZ: goto no_save;  /* will raise an error next loop */
395           case 'z': {  /* zap following span of spaces */
396             luaZ_buffremove(ls->buff, 1);  /* remove '\\' */
397             next(ls);  /* skip the 'z' */
398             while (lisspace(ls->current)) {
399               if (currIsNewline(ls)) inclinenumber(ls);
400               else next(ls);
401             }
402             goto no_save;
403           }
404           default: {
405             esccheck(ls, lisdigit(ls->current), "invalid escape sequence");
406             c = readdecesc(ls);  /* digital escape '\ddd' */
407             goto only_save;
408           }
409         }
410        read_save:
411          next(ls);
412          /* go through */
413        only_save:
414          luaZ_buffremove(ls->buff, 1);  /* remove '\\' */
415          save(ls, c);
416          /* go through */
417        no_save: break;
418       }
419       default:
420         save_and_next(ls);
421     }
422   }
423   save_and_next(ls);  /* skip delimiter */
424   seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1,
425                                    luaZ_bufflen(ls->buff) - 2);
426 }
427 
428 
llex(LexState * ls,SemInfo * seminfo)429 static int llex (LexState *ls, SemInfo *seminfo) {
430   luaZ_resetbuffer(ls->buff);
431   for (;;) {
432     switch (ls->current) {
433       case '\n': case '\r': {  /* line breaks */
434         inclinenumber(ls);
435         break;
436       }
437       case ' ': case '\f': case '\t': case '\v': {  /* spaces */
438         next(ls);
439         break;
440       }
441       case '-': {  /* '-' or '--' (comment) */
442         next(ls);
443         if (ls->current != '-') return '-';
444         /* else is a comment */
445         next(ls);
446         if (ls->current == '[') {  /* long comment? */
447           int sep = skip_sep(ls);
448           luaZ_resetbuffer(ls->buff);  /* 'skip_sep' may dirty the buffer */
449           if (sep >= 0) {
450             read_long_string(ls, NULL, sep);  /* skip long comment */
451             luaZ_resetbuffer(ls->buff);  /* previous call may dirty the buff. */
452             break;
453           }
454         }
455         /* else short comment */
456         while (!currIsNewline(ls) && ls->current != EOZ)
457           next(ls);  /* skip until end of line (or end of file) */
458         break;
459       }
460       case '[': {  /* long string or simply '[' */
461         int sep = skip_sep(ls);
462         if (sep >= 0) {
463           read_long_string(ls, seminfo, sep);
464           return TK_STRING;
465         }
466         else if (sep != -1)  /* '[=...' missing second bracket */
467           lexerror(ls, "invalid long string delimiter", TK_STRING);
468         return '[';
469       }
470       case '=': {
471         next(ls);
472         if (check_next1(ls, '=')) return TK_EQ;
473         else return '=';
474       }
475       case '<': {
476         next(ls);
477         if (check_next1(ls, '=')) return TK_LE;
478         else if (check_next1(ls, '<')) return TK_SHL;
479         else return '<';
480       }
481       case '>': {
482         next(ls);
483         if (check_next1(ls, '=')) return TK_GE;
484         else if (check_next1(ls, '>')) return TK_SHR;
485         else return '>';
486       }
487       case '/': {
488         next(ls);
489         if (check_next1(ls, '/')) return TK_IDIV;
490         else return '/';
491       }
492       case '~': {
493         next(ls);
494         if (check_next1(ls, '=')) return TK_NE;
495         else return '~';
496       }
497       case ':': {
498         next(ls);
499         if (check_next1(ls, ':')) return TK_DBCOLON;
500         else return ':';
501       }
502       case '"': case '\'': {  /* short literal strings */
503         read_string(ls, ls->current, seminfo);
504         return TK_STRING;
505       }
506       case '.': {  /* '.', '..', '...', or number */
507         save_and_next(ls);
508         if (check_next1(ls, '.')) {
509           if (check_next1(ls, '.'))
510             return TK_DOTS;   /* '...' */
511           else return TK_CONCAT;   /* '..' */
512         }
513         else if (!lisdigit(ls->current)) return '.';
514         else return read_numeral(ls, seminfo);
515       }
516       case '0': case '1': case '2': case '3': case '4':
517       case '5': case '6': case '7': case '8': case '9': {
518         return read_numeral(ls, seminfo);
519       }
520       case EOZ: {
521         return TK_EOS;
522       }
523       default: {
524         if (lislalpha(ls->current)) {  /* identifier or reserved word? */
525           TString *ts;
526           do {
527             save_and_next(ls);
528           } while (lislalnum(ls->current));
529           ts = luaX_newstring(ls, luaZ_buffer(ls->buff),
530                                   luaZ_bufflen(ls->buff));
531           seminfo->ts = ts;
532           if (isreserved(ts))  /* reserved word? */
533             return ts->extra - 1 + FIRST_RESERVED;
534           else {
535             return TK_NAME;
536           }
537         }
538         else {  /* single-char tokens (+ - / ...) */
539           int c = ls->current;
540           next(ls);
541           return c;
542         }
543       }
544     }
545   }
546 }
547 
548 
luaX_next(LexState * ls)549 void luaX_next (LexState *ls) {
550   ls->lastline = ls->linenumber;
551   if (ls->lookahead.token != TK_EOS) {  /* is there a look-ahead token? */
552     ls->t = ls->lookahead;  /* use this one */
553     ls->lookahead.token = TK_EOS;  /* and discharge it */
554   }
555   else
556     ls->t.token = llex(ls, &ls->t.seminfo);  /* read next token */
557 }
558 
559 
luaX_lookahead(LexState * ls)560 int luaX_lookahead (LexState *ls) {
561   lua_assert(ls->lookahead.token == TK_EOS);
562   ls->lookahead.token = llex(ls, &ls->lookahead.seminfo);
563   return ls->lookahead.token;
564 }
565 
566