Lines Matching full:token
23 Token::Type FindDelim(char c) in FindDelim()
26 static const std::unordered_map<char, Token::Type> DELIM = {{',', Token::Type::DEL_COMMA}, in FindDelim()
27 {':', Token::Type::DEL_COLON}, in FindDelim()
28 {'{', Token::Type::DEL_BRACE_L}, in FindDelim()
29 {'}', Token::Type::DEL_BRACE_R}, in FindDelim()
30 {'(', Token::Type::DEL_BRACKET_L}, in FindDelim()
31 {')', Token::Type::DEL_BRACKET_R}, in FindDelim()
32 {'<', Token::Type::DEL_LT}, in FindDelim()
33 {'>', Token::Type::DEL_GT}, in FindDelim()
34 {'=', Token::Type::DEL_EQ}, in FindDelim()
35 … {'[', Token::Type::DEL_SQUARE_BRACKET_L}, in FindDelim()
36 … {']', Token::Type::DEL_SQUARE_BRACKET_R}}; in FindDelim()
41 return Token::Type::ID_BAD; in FindDelim()
47 Token::Type FindOperation(std::string_view s) in FindOperation()
50 static const std::unordered_map<std::string_view, Token::Type> OPERATIONS = { in FindOperation()
53 {std::string_view(name), Token::Type::ID_OP_##inst_code}, in FindOperation()
61 return Token::Type::ID_BAD; in FindOperation()
67 Token::Type Findkeyword(std::string_view s) in Findkeyword()
70 static const std::unordered_map<std::string_view, Token::Type> KEYWORDS = { in Findkeyword()
72 #define KEYWORDS(name, inst_code) {std::string_view(name), Token::Type::ID_##inst_code}, in Findkeyword()
80 return Token::Type::ID_BAD; in Findkeyword()
86 std::string_view TokenTypeWhat(Token::Type t) in TokenTypeWhat()
88 if (t >= Token::Type::OPERATION && t < Token::Type::KEYWORD) { in TokenTypeWhat()
92 if (t >= Token::Type::KEYWORD) { in TokenTypeWhat()
97 case Token::Type::ID_BAD: { in TokenTypeWhat()
100 case Token::Type::DEL_COMMA: { in TokenTypeWhat()
103 case Token::Type::DEL_COLON: { in TokenTypeWhat()
106 case Token::Type::DEL_BRACE_L: { in TokenTypeWhat()
109 case Token::Type::DEL_BRACE_R: { in TokenTypeWhat()
112 case Token::Type::DEL_BRACKET_L: { in TokenTypeWhat()
115 case Token::Type::DEL_BRACKET_R: { in TokenTypeWhat()
118 case Token::Type::DEL_SQUARE_BRACKET_L: { in TokenTypeWhat()
121 case Token::Type::DEL_SQUARE_BRACKET_R: { in TokenTypeWhat()
124 case Token::Type::DEL_GT: { in TokenTypeWhat()
127 case Token::Type::DEL_LT: { in TokenTypeWhat()
130 case Token::Type::DEL_EQ: { in TokenTypeWhat()
133 case Token::Type::DEL_DOT: { in TokenTypeWhat()
136 case Token::Type::ID: { in TokenTypeWhat()
139 case Token::Type::ID_STRING: { in TokenTypeWhat()
187 return std::pair<std::vector<Token>, Error>(lines_.back().tokens, err_); in TokenizeString()
196 /* Return the type of token */
197 Token::Type Lexer::LexGetType(size_t beg, size_t end) const in LexGetType()
199 if (FindDelim(curr_line_->buffer[beg]) != Token::Type::ID_BAD) { /* delimiter */ in LexGetType()
205 Token::Type type = Findkeyword(p); in LexGetType()
207 if (type != Token::Type::ID_BAD) { in LexGetType()
213 if (type != Token::Type::ID_BAD) { in LexGetType()
218 return Token::Type::ID_STRING; in LexGetType()
221 return Token::Type::ID; /* other */ in LexGetType()
264 * to the first and last characters of a corresponding token.
281 LOG(DEBUG, ASSEMBLER) << "token search started (line " << lines_.size() << "): " in LexTokens()
300 if (FindDelim(curr_line_->buffer[curr_line_->pos]) != Token::Type::ID_BAD) { in LexTokens()
307 … while (!Eol() && FindDelim(curr_line_->buffer[curr_line_->pos]) == Token::Type::ID_BAD && in LexTokens()
315 LOG(DEBUG, ASSEMBLER) << "token identified (line " << lines_.size() << ", " in LexTokens()
316 << "token " << curr_line_->tokens.size() + 1 << "): " in LexTokens()