Lines Matching full:token
19 #include "compiler/preprocessor/Token.h"
45 DirectiveType getDirective(const pp::Token *token) in getDirective() argument
61 if (token->type != pp::Token::IDENTIFIER) in getDirective()
64 if (token->text == kDirectiveDefine) in getDirective()
66 if (token->text == kDirectiveUndef) in getDirective()
68 if (token->text == kDirectiveIf) in getDirective()
70 if (token->text == kDirectiveIfdef) in getDirective()
72 if (token->text == kDirectiveIfndef) in getDirective()
74 if (token->text == kDirectiveElse) in getDirective()
76 if (token->text == kDirectiveElif) in getDirective()
78 if (token->text == kDirectiveEndif) in getDirective()
80 if (token->text == kDirectiveError) in getDirective()
82 if (token->text == kDirectivePragma) in getDirective()
84 if (token->text == kDirectiveExtension) in getDirective()
86 if (token->text == kDirectiveVersion) in getDirective()
88 if (token->text == kDirectiveLine) in getDirective()
110 // Returns true if the token represents End Of Directive.
111 bool isEOD(const pp::Token *token) in isEOD() argument
113 return (token->type == '\n') || (token->type == pp::Token::LAST); in isEOD()
116 void skipUntilEOD(pp::Lexer *lexer, pp::Token *token) in skipUntilEOD() argument
118 while (!isEOD(token)) in skipUntilEOD()
120 lexer->lex(token); in skipUntilEOD()
163 void DirectiveParser::lex(Token *token) in lex() argument
167 mTokenizer->lex(token); in lex()
169 if (token->type == Token::PP_HASH) in lex()
171 parseDirective(token); in lex()
174 else if (!isEOD(token) && !skipping()) in lex()
179 // If #version does not appear before first token, then this is in lex()
181 handleVersion(token->location); in lex()
185 if (token->type == Token::LAST) in lex()
196 } while (skipping() || (token->type == '\n')); in lex()
201 void DirectiveParser::parseDirective(Token *token) in parseDirective() argument
203 ASSERT(token->type == Token::PP_HASH); in parseDirective()
205 mTokenizer->lex(token); in parseDirective()
206 if (isEOD(token)) in parseDirective()
212 DirectiveType directive = getDirective(token); in parseDirective()
218 handleVersion(token->location); in parseDirective()
225 skipUntilEOD(mTokenizer, token); in parseDirective()
232 mDiagnostics->report(Diagnostics::PP_DIRECTIVE_INVALID_NAME, token->location, in parseDirective()
233 token->text); in parseDirective()
234 skipUntilEOD(mTokenizer, token); in parseDirective()
237 parseDefine(token); in parseDirective()
240 parseUndef(token); in parseDirective()
243 parseIf(token); in parseDirective()
246 parseIfdef(token); in parseDirective()
249 parseIfndef(token); in parseDirective()
252 parseElse(token); in parseDirective()
255 parseElif(token); in parseDirective()
258 parseEndif(token); in parseDirective()
261 parseError(token); in parseDirective()
264 parsePragma(token); in parseDirective()
267 parseExtension(token); in parseDirective()
270 parseVersion(token); in parseDirective()
273 parseLine(token); in parseDirective()
280 skipUntilEOD(mTokenizer, token); in parseDirective()
281 if (token->type == Token::LAST) in parseDirective()
283 mDiagnostics->report(Diagnostics::PP_EOF_IN_DIRECTIVE, token->location, token->text); in parseDirective()
287 void DirectiveParser::parseDefine(Token *token) in parseDefine() argument
289 ASSERT(getDirective(token) == DIRECTIVE_DEFINE); in parseDefine()
291 mTokenizer->lex(token); in parseDefine()
292 if (token->type != Token::IDENTIFIER) in parseDefine()
294 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text); in parseDefine()
297 if (isMacroPredefined(token->text, *mMacroSet)) in parseDefine()
299 mDiagnostics->report(Diagnostics::PP_MACRO_PREDEFINED_REDEFINED, token->location, in parseDefine()
300 token->text); in parseDefine()
303 if (isMacroNameReserved(token->text)) in parseDefine()
305 mDiagnostics->report(Diagnostics::PP_MACRO_NAME_RESERVED, token->location, token->text); in parseDefine()
313 if (hasDoubleUnderscores(token->text)) in parseDefine()
315 mDiagnostics->report(Diagnostics::PP_WARNING_MACRO_NAME_RESERVED, token->location, in parseDefine()
316 token->text); in parseDefine()
321 macro->name = token->text; in parseDefine()
323 mTokenizer->lex(token); in parseDefine()
324 if (token->type == '(' && !token->hasLeadingSpace()) in parseDefine()
330 mTokenizer->lex(token); in parseDefine()
331 if (token->type != Token::IDENTIFIER) in parseDefine()
334 if (std::find(macro->parameters.begin(), macro->parameters.end(), token->text) != in parseDefine()
338 token->location, token->text); in parseDefine()
342 macro->parameters.push_back(token->text); in parseDefine()
344 mTokenizer->lex(token); // Get ','. in parseDefine()
345 } while (token->type == ','); in parseDefine()
347 if (token->type != ')') in parseDefine()
349 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text); in parseDefine()
352 mTokenizer->lex(token); // Get ')'. in parseDefine()
355 while ((token->type != '\n') && (token->type != Token::LAST)) in parseDefine()
357 // Reset the token location because it is unnecessary in replacement in parseDefine()
358 // list. Resetting it also allows us to reuse Token::equals() to in parseDefine()
360 token->location = SourceLocation(); in parseDefine()
361 macro->replacements.push_back(*token); in parseDefine()
362 mTokenizer->lex(token); in parseDefine()
375 mDiagnostics->report(Diagnostics::PP_MACRO_REDEFINED, token->location, macro->name); in parseDefine()
381 void DirectiveParser::parseUndef(Token *token) in parseUndef() argument
383 ASSERT(getDirective(token) == DIRECTIVE_UNDEF); in parseUndef()
385 mTokenizer->lex(token); in parseUndef()
386 if (token->type != Token::IDENTIFIER) in parseUndef()
388 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text); in parseUndef()
392 MacroSet::iterator iter = mMacroSet->find(token->text); in parseUndef()
397 mDiagnostics->report(Diagnostics::PP_MACRO_PREDEFINED_UNDEFINED, token->location, in parseUndef()
398 token->text); in parseUndef()
403 mDiagnostics->report(Diagnostics::PP_MACRO_UNDEFINED_WHILE_INVOKED, token->location, in parseUndef()
404 token->text); in parseUndef()
413 mTokenizer->lex(token); in parseUndef()
414 if (!isEOD(token)) in parseUndef()
416 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text); in parseUndef()
417 skipUntilEOD(mTokenizer, token); in parseUndef()
421 void DirectiveParser::parseIf(Token *token) in parseIf() argument
423 ASSERT(getDirective(token) == DIRECTIVE_IF); in parseIf()
424 parseConditionalIf(token); in parseIf()
427 void DirectiveParser::parseIfdef(Token *token) in parseIfdef() argument
429 ASSERT(getDirective(token) == DIRECTIVE_IFDEF); in parseIfdef()
430 parseConditionalIf(token); in parseIfdef()
433 void DirectiveParser::parseIfndef(Token *token) in parseIfndef() argument
435 ASSERT(getDirective(token) == DIRECTIVE_IFNDEF); in parseIfndef()
436 parseConditionalIf(token); in parseIfndef()
439 void DirectiveParser::parseElse(Token *token) in parseElse() argument
441 ASSERT(getDirective(token) == DIRECTIVE_ELSE); in parseElse()
445 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_ELSE_WITHOUT_IF, token->location, in parseElse()
446 token->text); in parseElse()
447 skipUntilEOD(mTokenizer, token); in parseElse()
455 skipUntilEOD(mTokenizer, token); in parseElse()
460 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_ELSE_AFTER_ELSE, token->location, in parseElse()
461 token->text); in parseElse()
462 skipUntilEOD(mTokenizer, token); in parseElse()
471 mTokenizer->lex(token); in parseElse()
472 if (!isEOD(token)) in parseElse()
474 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN, token->location, in parseElse()
475 token->text); in parseElse()
476 skipUntilEOD(mTokenizer, token); in parseElse()
480 void DirectiveParser::parseElif(Token *token) in parseElif() argument
482 ASSERT(getDirective(token) == DIRECTIVE_ELIF); in parseElif()
486 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_ELIF_WITHOUT_IF, token->location, in parseElif()
487 token->text); in parseElif()
488 skipUntilEOD(mTokenizer, token); in parseElif()
496 skipUntilEOD(mTokenizer, token); in parseElif()
501 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_ELIF_AFTER_ELSE, token->location, in parseElif()
502 token->text); in parseElif()
503 skipUntilEOD(mTokenizer, token); in parseElif()
511 skipUntilEOD(mTokenizer, token); in parseElif()
515 int expression = parseExpressionIf(token); in parseElif()
520 void DirectiveParser::parseEndif(Token *token) in parseEndif() argument
522 ASSERT(getDirective(token) == DIRECTIVE_ENDIF); in parseEndif()
526 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_ENDIF_WITHOUT_IF, token->location, in parseEndif()
527 token->text); in parseEndif()
528 skipUntilEOD(mTokenizer, token); in parseEndif()
535 mTokenizer->lex(token); in parseEndif()
536 if (!isEOD(token)) in parseEndif()
538 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN, token->location, in parseEndif()
539 token->text); in parseEndif()
540 skipUntilEOD(mTokenizer, token); in parseEndif()
544 void DirectiveParser::parseError(Token *token) in parseError() argument
546 ASSERT(getDirective(token) == DIRECTIVE_ERROR); in parseError()
549 mTokenizer->lex(token); in parseError()
550 while ((token->type != '\n') && (token->type != Token::LAST)) in parseError()
552 stream << *token; in parseError()
553 mTokenizer->lex(token); in parseError()
555 mDirectiveHandler->handleError(token->location, stream.str()); in parseError()
559 void DirectiveParser::parsePragma(Token *token) in parsePragma() argument
561 ASSERT(getDirective(token) == DIRECTIVE_PRAGMA); in parsePragma()
575 mTokenizer->lex(token); in parsePragma()
576 bool stdgl = token->text == "STDGL"; in parsePragma()
579 mTokenizer->lex(token); in parsePragma()
581 while ((token->type != '\n') && (token->type != Token::LAST)) in parsePragma()
586 name = token->text; in parsePragma()
587 valid = valid && (token->type == Token::IDENTIFIER); in parsePragma()
590 valid = valid && (token->type == '('); in parsePragma()
593 value = token->text; in parsePragma()
594 valid = valid && (token->type == Token::IDENTIFIER); in parsePragma()
597 valid = valid && (token->type == ')'); in parsePragma()
603 mTokenizer->lex(token); in parsePragma()
611 mDiagnostics->report(Diagnostics::PP_UNRECOGNIZED_PRAGMA, token->location, name); in parsePragma()
615 mDirectiveHandler->handlePragma(token->location, name, value, stdgl); in parsePragma()
619 void DirectiveParser::parseExtension(Token *token) in parseExtension() argument
621 ASSERT(getDirective(token) == DIRECTIVE_EXTENSION); in parseExtension()
634 mTokenizer->lex(token); in parseExtension()
635 while ((token->type != '\n') && (token->type != Token::LAST)) in parseExtension()
640 if (valid && (token->type != Token::IDENTIFIER)) in parseExtension()
642 mDiagnostics->report(Diagnostics::PP_INVALID_EXTENSION_NAME, token->location, in parseExtension()
643 token->text); in parseExtension()
647 name = token->text; in parseExtension()
650 if (valid && (token->type != ':')) in parseExtension()
652 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, in parseExtension()
653 token->text); in parseExtension()
658 if (valid && (token->type != Token::IDENTIFIER)) in parseExtension()
661 token->location, token->text); in parseExtension()
665 behavior = token->text; in parseExtension()
670 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, in parseExtension()
671 token->text); in parseExtension()
676 mTokenizer->lex(token); in parseExtension()
680 mDiagnostics->report(Diagnostics::PP_INVALID_EXTENSION_DIRECTIVE, token->location, in parseExtension()
681 token->text); in parseExtension()
689 token->location, token->text); in parseExtension()
697 token->location, token->text); in parseExtension()
702 token->location, token->text); in parseExtension()
711 mDirectiveHandler->handleExtension(token->location, name, behavior); in parseExtension()
714 void DirectiveParser::parseVersion(Token *token) in parseVersion() argument
716 ASSERT(getDirective(token) == DIRECTIVE_VERSION); in parseVersion()
720 mDiagnostics->report(Diagnostics::PP_VERSION_NOT_FIRST_STATEMENT, token->location, in parseVersion()
721 token->text); in parseVersion()
722 skipUntilEOD(mTokenizer, token); in parseVersion()
737 mTokenizer->lex(token); in parseVersion()
738 while (valid && (token->type != '\n') && (token->type != Token::LAST)) in parseVersion()
743 if (token->type != Token::CONST_INT) in parseVersion()
745 mDiagnostics->report(Diagnostics::PP_INVALID_VERSION_NUMBER, token->location, in parseVersion()
746 token->text); in parseVersion()
749 if (valid && !token->iValue(&version)) in parseVersion()
751 mDiagnostics->report(Diagnostics::PP_INTEGER_OVERFLOW, token->location, in parseVersion()
752 token->text); in parseVersion()
768 if (token->type != Token::IDENTIFIER || token->text != "es") in parseVersion()
770 mDiagnostics->report(Diagnostics::PP_INVALID_VERSION_DIRECTIVE, token->location, in parseVersion()
771 token->text); in parseVersion()
777 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, in parseVersion()
778 token->text); in parseVersion()
783 mTokenizer->lex(token); in parseVersion()
788 mDiagnostics->report(Diagnostics::PP_INVALID_VERSION_DIRECTIVE, token->location, in parseVersion()
789 token->text); in parseVersion()
793 if (valid && version >= 300 && token->location.line > 1) in parseVersion()
795 mDiagnostics->report(Diagnostics::PP_VERSION_NOT_FIRST_LINE_ESSL3, token->location, in parseVersion()
796 token->text); in parseVersion()
803 handleVersion(token->location); in parseVersion()
807 void DirectiveParser::parseLine(Token *token) in parseLine() argument
809 ASSERT(getDirective(token) == DIRECTIVE_LINE); in parseLine()
817 // Lex the first token after "#line" so we can check it for EOD. in parseLine()
818 macroExpander.lex(token); in parseLine()
820 if (isEOD(token)) in parseLine()
822 mDiagnostics->report(Diagnostics::PP_INVALID_LINE_DIRECTIVE, token->location, token->text); in parseLine()
834 // The first token was lexed earlier to check if it was EOD. Include in parseLine()
835 // the token in parsing for a second time by setting the in parseLine()
837 expressionParser.parse(token, &line, true, errorSettings, &valid); in parseLine()
838 if (!isEOD(token) && valid) in parseLine()
842 // advanced to the first token of the file expression - this is the in parseLine()
843 // token that makes the parser reduce the "input" rule for the line in parseLine()
846 expressionParser.parse(token, &file, true, errorSettings, &valid); in parseLine()
849 if (!isEOD(token)) in parseLine()
853 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, in parseLine()
854 token->text); in parseLine()
857 skipUntilEOD(mTokenizer, token); in parseLine()
878 void DirectiveParser::parseConditionalIf(Token *token) in parseConditionalIf() argument
881 block.type = token->text; in parseConditionalIf()
882 block.location = token->location; in parseConditionalIf()
890 skipUntilEOD(mTokenizer, token); in parseConditionalIf()
895 DirectiveType directive = getDirective(token); in parseConditionalIf()
901 expression = parseExpressionIf(token); in parseConditionalIf()
904 expression = parseExpressionIfdef(token); in parseConditionalIf()
907 expression = parseExpressionIfdef(token) == 0 ? 1 : 0; in parseConditionalIf()
919 int DirectiveParser::parseExpressionIf(Token *token) in parseExpressionIf() argument
921 ASSERT((getDirective(token) == DIRECTIVE_IF) || (getDirective(token) == DIRECTIVE_ELIF)); in parseExpressionIf()
932 expressionParser.parse(token, &expression, false, errorSettings, &valid); in parseExpressionIf()
935 if (!isEOD(token)) in parseExpressionIf()
937 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN, token->location, in parseExpressionIf()
938 token->text); in parseExpressionIf()
939 skipUntilEOD(mTokenizer, token); in parseExpressionIf()
945 int DirectiveParser::parseExpressionIfdef(Token *token) in parseExpressionIfdef() argument
947 ASSERT((getDirective(token) == DIRECTIVE_IFDEF) || (getDirective(token) == DIRECTIVE_IFNDEF)); in parseExpressionIfdef()
949 mTokenizer->lex(token); in parseExpressionIfdef()
950 if (token->type != Token::IDENTIFIER) in parseExpressionIfdef()
952 mDiagnostics->report(Diagnostics::PP_UNEXPECTED_TOKEN, token->location, token->text); in parseExpressionIfdef()
953 skipUntilEOD(mTokenizer, token); in parseExpressionIfdef()
957 MacroSet::const_iterator iter = mMacroSet->find(token->text); in parseExpressionIfdef()
961 mTokenizer->lex(token); in parseExpressionIfdef()
962 if (!isEOD(token)) in parseExpressionIfdef()
964 mDiagnostics->report(Diagnostics::PP_CONDITIONAL_UNEXPECTED_TOKEN, token->location, in parseExpressionIfdef()
965 token->text); in parseExpressionIfdef()
966 skipUntilEOD(mTokenizer, token); in parseExpressionIfdef()