1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Implement the Lexer for TableGen.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "TGLexer.h"
15 #include "llvm/ADT/StringSwitch.h"
16 #include "llvm/ADT/Twine.h"
17 #include "llvm/Config/config.h" // for strtoull()/strtoll() define
18 #include "llvm/Support/MemoryBuffer.h"
19 #include "llvm/Support/SourceMgr.h"
20 #include "llvm/TableGen/Error.h"
21 #include <cctype>
22 #include <cerrno>
23 #include <cstdio>
24 #include <cstdlib>
25 #include <cstring>
26
27 using namespace llvm;
28
TGLexer(SourceMgr & SM)29 TGLexer::TGLexer(SourceMgr &SM) : SrcMgr(SM) {
30 CurBuffer = SrcMgr.getMainFileID();
31 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
32 CurPtr = CurBuf.begin();
33 TokStart = nullptr;
34 }
35
getLoc() const36 SMLoc TGLexer::getLoc() const {
37 return SMLoc::getFromPointer(TokStart);
38 }
39
40 /// ReturnError - Set the error to the specified string at the specified
41 /// location. This is defined to always return tgtok::Error.
ReturnError(const char * Loc,const Twine & Msg)42 tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
43 PrintError(Loc, Msg);
44 return tgtok::Error;
45 }
46
getNextChar()47 int TGLexer::getNextChar() {
48 char CurChar = *CurPtr++;
49 switch (CurChar) {
50 default:
51 return (unsigned char)CurChar;
52 case 0: {
53 // A nul character in the stream is either the end of the current buffer or
54 // a random nul in the file. Disambiguate that here.
55 if (CurPtr-1 != CurBuf.end())
56 return 0; // Just whitespace.
57
58 // If this is the end of an included file, pop the parent file off the
59 // include stack.
60 SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
61 if (ParentIncludeLoc != SMLoc()) {
62 CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
63 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
64 CurPtr = ParentIncludeLoc.getPointer();
65 return getNextChar();
66 }
67
68 // Otherwise, return end of file.
69 --CurPtr; // Another call to lex will return EOF again.
70 return EOF;
71 }
72 case '\n':
73 case '\r':
74 // Handle the newline character by ignoring it and incrementing the line
75 // count. However, be careful about 'dos style' files with \n\r in them.
76 // Only treat a \n\r or \r\n as a single line.
77 if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
78 *CurPtr != CurChar)
79 ++CurPtr; // Eat the two char newline sequence.
80 return '\n';
81 }
82 }
83
peekNextChar(int Index)84 int TGLexer::peekNextChar(int Index) {
85 return *(CurPtr + Index);
86 }
87
LexToken()88 tgtok::TokKind TGLexer::LexToken() {
89 TokStart = CurPtr;
90 // This always consumes at least one character.
91 int CurChar = getNextChar();
92
93 switch (CurChar) {
94 default:
95 // Handle letters: [a-zA-Z_]
96 if (isalpha(CurChar) || CurChar == '_')
97 return LexIdentifier();
98
99 // Unknown character, emit an error.
100 return ReturnError(TokStart, "Unexpected character");
101 case EOF: return tgtok::Eof;
102 case ':': return tgtok::colon;
103 case ';': return tgtok::semi;
104 case '.': return tgtok::period;
105 case ',': return tgtok::comma;
106 case '<': return tgtok::less;
107 case '>': return tgtok::greater;
108 case ']': return tgtok::r_square;
109 case '{': return tgtok::l_brace;
110 case '}': return tgtok::r_brace;
111 case '(': return tgtok::l_paren;
112 case ')': return tgtok::r_paren;
113 case '=': return tgtok::equal;
114 case '?': return tgtok::question;
115 case '#': return tgtok::paste;
116
117 case 0:
118 case ' ':
119 case '\t':
120 case '\n':
121 case '\r':
122 // Ignore whitespace.
123 return LexToken();
124 case '/':
125 // If this is the start of a // comment, skip until the end of the line or
126 // the end of the buffer.
127 if (*CurPtr == '/')
128 SkipBCPLComment();
129 else if (*CurPtr == '*') {
130 if (SkipCComment())
131 return tgtok::Error;
132 } else // Otherwise, this is an error.
133 return ReturnError(TokStart, "Unexpected character");
134 return LexToken();
135 case '-': case '+':
136 case '0': case '1': case '2': case '3': case '4': case '5': case '6':
137 case '7': case '8': case '9': {
138 int NextChar = 0;
139 if (isdigit(CurChar)) {
140 // Allow identifiers to start with a number if it is followed by
141 // an identifier. This can happen with paste operations like
142 // foo#8i.
143 int i = 0;
144 do {
145 NextChar = peekNextChar(i++);
146 } while (isdigit(NextChar));
147
148 if (NextChar == 'x' || NextChar == 'b') {
149 // If this is [0-9]b[01] or [0-9]x[0-9A-fa-f] this is most
150 // likely a number.
151 int NextNextChar = peekNextChar(i);
152 switch (NextNextChar) {
153 default:
154 break;
155 case '0': case '1':
156 if (NextChar == 'b')
157 return LexNumber();
158 // Fallthrough
159 case '2': case '3': case '4': case '5':
160 case '6': case '7': case '8': case '9':
161 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
162 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
163 if (NextChar == 'x')
164 return LexNumber();
165 break;
166 }
167 }
168 }
169
170 if (isalpha(NextChar) || NextChar == '_')
171 return LexIdentifier();
172
173 return LexNumber();
174 }
175 case '"': return LexString();
176 case '$': return LexVarName();
177 case '[': return LexBracket();
178 case '!': return LexExclaim();
179 }
180 }
181
182 /// LexString - Lex "[^"]*"
LexString()183 tgtok::TokKind TGLexer::LexString() {
184 const char *StrStart = CurPtr;
185
186 CurStrVal = "";
187
188 while (*CurPtr != '"') {
189 // If we hit the end of the buffer, report an error.
190 if (*CurPtr == 0 && CurPtr == CurBuf.end())
191 return ReturnError(StrStart, "End of file in string literal");
192
193 if (*CurPtr == '\n' || *CurPtr == '\r')
194 return ReturnError(StrStart, "End of line in string literal");
195
196 if (*CurPtr != '\\') {
197 CurStrVal += *CurPtr++;
198 continue;
199 }
200
201 ++CurPtr;
202
203 switch (*CurPtr) {
204 case '\\': case '\'': case '"':
205 // These turn into their literal character.
206 CurStrVal += *CurPtr++;
207 break;
208 case 't':
209 CurStrVal += '\t';
210 ++CurPtr;
211 break;
212 case 'n':
213 CurStrVal += '\n';
214 ++CurPtr;
215 break;
216
217 case '\n':
218 case '\r':
219 return ReturnError(CurPtr, "escaped newlines not supported in tblgen");
220
221 // If we hit the end of the buffer, report an error.
222 case '\0':
223 if (CurPtr == CurBuf.end())
224 return ReturnError(StrStart, "End of file in string literal");
225 // FALL THROUGH
226 default:
227 return ReturnError(CurPtr, "invalid escape in string literal");
228 }
229 }
230
231 ++CurPtr;
232 return tgtok::StrVal;
233 }
234
LexVarName()235 tgtok::TokKind TGLexer::LexVarName() {
236 if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
237 return ReturnError(TokStart, "Invalid variable name");
238
239 // Otherwise, we're ok, consume the rest of the characters.
240 const char *VarNameStart = CurPtr++;
241
242 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
243 ++CurPtr;
244
245 CurStrVal.assign(VarNameStart, CurPtr);
246 return tgtok::VarName;
247 }
248
249
LexIdentifier()250 tgtok::TokKind TGLexer::LexIdentifier() {
251 // The first letter is [a-zA-Z_#].
252 const char *IdentStart = TokStart;
253
254 // Match the rest of the identifier regex: [0-9a-zA-Z_#]*
255 while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
256 ++CurPtr;
257
258 // Check to see if this identifier is a keyword.
259 StringRef Str(IdentStart, CurPtr-IdentStart);
260
261 if (Str == "include") {
262 if (LexInclude()) return tgtok::Error;
263 return Lex();
264 }
265
266 tgtok::TokKind Kind = StringSwitch<tgtok::TokKind>(Str)
267 .Case("int", tgtok::Int)
268 .Case("bit", tgtok::Bit)
269 .Case("bits", tgtok::Bits)
270 .Case("string", tgtok::String)
271 .Case("list", tgtok::List)
272 .Case("code", tgtok::Code)
273 .Case("dag", tgtok::Dag)
274 .Case("class", tgtok::Class)
275 .Case("def", tgtok::Def)
276 .Case("foreach", tgtok::Foreach)
277 .Case("defm", tgtok::Defm)
278 .Case("multiclass", tgtok::MultiClass)
279 .Case("field", tgtok::Field)
280 .Case("let", tgtok::Let)
281 .Case("in", tgtok::In)
282 .Default(tgtok::Id);
283
284 if (Kind == tgtok::Id)
285 CurStrVal.assign(Str.begin(), Str.end());
286 return Kind;
287 }
288
289 /// LexInclude - We just read the "include" token. Get the string token that
290 /// comes next and enter the include.
LexInclude()291 bool TGLexer::LexInclude() {
292 // The token after the include must be a string.
293 tgtok::TokKind Tok = LexToken();
294 if (Tok == tgtok::Error) return true;
295 if (Tok != tgtok::StrVal) {
296 PrintError(getLoc(), "Expected filename after include");
297 return true;
298 }
299
300 // Get the string.
301 std::string Filename = CurStrVal;
302 std::string IncludedFile;
303
304
305 CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
306 IncludedFile);
307 if (!CurBuffer) {
308 PrintError(getLoc(), "Could not find include file '" + Filename + "'");
309 return true;
310 }
311
312 DependenciesMapTy::const_iterator Found = Dependencies.find(IncludedFile);
313 if (Found != Dependencies.end()) {
314 PrintError(getLoc(),
315 "File '" + IncludedFile + "' has already been included.");
316 SrcMgr.PrintMessage(Found->second, SourceMgr::DK_Note,
317 "previously included here");
318 return true;
319 }
320 Dependencies.insert(std::make_pair(IncludedFile, getLoc()));
321 // Save the line number and lex buffer of the includer.
322 CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
323 CurPtr = CurBuf.begin();
324 return false;
325 }
326
SkipBCPLComment()327 void TGLexer::SkipBCPLComment() {
328 ++CurPtr; // skip the second slash.
329 while (1) {
330 switch (*CurPtr) {
331 case '\n':
332 case '\r':
333 return; // Newline is end of comment.
334 case 0:
335 // If this is the end of the buffer, end the comment.
336 if (CurPtr == CurBuf.end())
337 return;
338 break;
339 }
340 // Otherwise, skip the character.
341 ++CurPtr;
342 }
343 }
344
345 /// SkipCComment - This skips C-style /**/ comments. The only difference from C
346 /// is that we allow nesting.
SkipCComment()347 bool TGLexer::SkipCComment() {
348 ++CurPtr; // skip the star.
349 unsigned CommentDepth = 1;
350
351 while (1) {
352 int CurChar = getNextChar();
353 switch (CurChar) {
354 case EOF:
355 PrintError(TokStart, "Unterminated comment!");
356 return true;
357 case '*':
358 // End of the comment?
359 if (CurPtr[0] != '/') break;
360
361 ++CurPtr; // End the */.
362 if (--CommentDepth == 0)
363 return false;
364 break;
365 case '/':
366 // Start of a nested comment?
367 if (CurPtr[0] != '*') break;
368 ++CurPtr;
369 ++CommentDepth;
370 break;
371 }
372 }
373 }
374
375 /// LexNumber - Lex:
376 /// [-+]?[0-9]+
377 /// 0x[0-9a-fA-F]+
378 /// 0b[01]+
LexNumber()379 tgtok::TokKind TGLexer::LexNumber() {
380 if (CurPtr[-1] == '0') {
381 if (CurPtr[0] == 'x') {
382 ++CurPtr;
383 const char *NumStart = CurPtr;
384 while (isxdigit(CurPtr[0]))
385 ++CurPtr;
386
387 // Requires at least one hex digit.
388 if (CurPtr == NumStart)
389 return ReturnError(TokStart, "Invalid hexadecimal number");
390
391 errno = 0;
392 CurIntVal = strtoll(NumStart, nullptr, 16);
393 if (errno == EINVAL)
394 return ReturnError(TokStart, "Invalid hexadecimal number");
395 if (errno == ERANGE) {
396 errno = 0;
397 CurIntVal = (int64_t)strtoull(NumStart, nullptr, 16);
398 if (errno == EINVAL)
399 return ReturnError(TokStart, "Invalid hexadecimal number");
400 if (errno == ERANGE)
401 return ReturnError(TokStart, "Hexadecimal number out of range");
402 }
403 return tgtok::IntVal;
404 } else if (CurPtr[0] == 'b') {
405 ++CurPtr;
406 const char *NumStart = CurPtr;
407 while (CurPtr[0] == '0' || CurPtr[0] == '1')
408 ++CurPtr;
409
410 // Requires at least one binary digit.
411 if (CurPtr == NumStart)
412 return ReturnError(CurPtr-2, "Invalid binary number");
413 CurIntVal = strtoll(NumStart, nullptr, 2);
414 return tgtok::BinaryIntVal;
415 }
416 }
417
418 // Check for a sign without a digit.
419 if (!isdigit(CurPtr[0])) {
420 if (CurPtr[-1] == '-')
421 return tgtok::minus;
422 else if (CurPtr[-1] == '+')
423 return tgtok::plus;
424 }
425
426 while (isdigit(CurPtr[0]))
427 ++CurPtr;
428 CurIntVal = strtoll(TokStart, nullptr, 10);
429 return tgtok::IntVal;
430 }
431
432 /// LexBracket - We just read '['. If this is a code block, return it,
433 /// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
LexBracket()434 tgtok::TokKind TGLexer::LexBracket() {
435 if (CurPtr[0] != '{')
436 return tgtok::l_square;
437 ++CurPtr;
438 const char *CodeStart = CurPtr;
439 while (1) {
440 int Char = getNextChar();
441 if (Char == EOF) break;
442
443 if (Char != '}') continue;
444
445 Char = getNextChar();
446 if (Char == EOF) break;
447 if (Char == ']') {
448 CurStrVal.assign(CodeStart, CurPtr-2);
449 return tgtok::CodeFragment;
450 }
451 }
452
453 return ReturnError(CodeStart-2, "Unterminated Code Block");
454 }
455
456 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
LexExclaim()457 tgtok::TokKind TGLexer::LexExclaim() {
458 if (!isalpha(*CurPtr))
459 return ReturnError(CurPtr - 1, "Invalid \"!operator\"");
460
461 const char *Start = CurPtr++;
462 while (isalpha(*CurPtr))
463 ++CurPtr;
464
465 // Check to see which operator this is.
466 tgtok::TokKind Kind =
467 StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
468 .Case("eq", tgtok::XEq)
469 .Case("if", tgtok::XIf)
470 .Case("head", tgtok::XHead)
471 .Case("tail", tgtok::XTail)
472 .Case("con", tgtok::XConcat)
473 .Case("add", tgtok::XADD)
474 .Case("and", tgtok::XAND)
475 .Case("shl", tgtok::XSHL)
476 .Case("sra", tgtok::XSRA)
477 .Case("srl", tgtok::XSRL)
478 .Case("cast", tgtok::XCast)
479 .Case("empty", tgtok::XEmpty)
480 .Case("subst", tgtok::XSubst)
481 .Case("foreach", tgtok::XForEach)
482 .Case("listconcat", tgtok::XListConcat)
483 .Case("strconcat", tgtok::XStrConcat)
484 .Default(tgtok::Error);
485
486 return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
487 }
488
489