Lines Matching refs:tokenize

44 from cpp import tokenize
549         if parts[-1].token_type == tokenize.NAME:
579         if (type_name and type_name[-1].token_type == tokenize.NAME and
580                 p.token_type == tokenize.NAME):
581           type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
749     if token.token_type == tokenize.NAME:
768         if next.token_type == tokenize.SYNTAX and next.name == '(':
773       syntax = tokenize.SYNTAX
783           new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
786           last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
797               self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
825     elif token.token_type == tokenize.SYNTAX:
831         if (token.token_type == tokenize.NAME and
835     elif token.token_type == tokenize.PREPROCESSOR:
906       if token.token_type != tokenize.PREPROCESSOR:
925       if token.token_type == tokenize.SYNTAX:
951     if token.whence == tokenize.WHENCE_STREAM:
952       token.whence = tokenize.WHENCE_QUEUE
955       assert token.whence == tokenize.WHENCE_QUEUE, token
960       if tokens[-1].whence == tokenize.WHENCE_STREAM:
962           token.whence = tokenize.WHENCE_QUEUE
965         assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
977     while (next_token.token_type == tokenize.NAME or
978            (next_token.token_type == tokenize.SYNTAX and
982       if last_token_was_name and next_token.token_type == tokenize.NAME:
984       last_token_was_name = next_token.token_type == tokenize.NAME
994     return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
1004       assert token.token_type == tokenize.SYNTAX, token
1010       assert token.token_type == tokenize.SYNTAX, token
1028       name = tokenize.Token(tokenize.NAME, 'operator[]',
1053     while token.token_type == tokenize.NAME:
1080     assert token.token_type == tokenize.SYNTAX, token
1100         assert token.token_type == tokenize.SYNTAX, token
1132           assert token.token_type == tokenize.CONSTANT, token
1178     seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
1185       if next and next.token_type == tokenize.SYNTAX:
1240     if token.token_type == tokenize.SYNTAX and token.name == ';':
1244     if token.token_type == tokenize.NAME and self._handling_typedef:
1252     if token.token_type == tokenize.SYNTAX and token.name == '{':
1258       if next.token_type != tokenize.NAME:
1264     assert token.token_type == tokenize.NAME, token
1273       is_syntax = (var_token.token_type == tokenize.SYNTAX and
1275       is_variable = (var_token.token_type == tokenize.NAME and
1281         if temp.token_type == tokenize.SYNTAX and temp.name == '(':
1284           struct = tokenize.Token(tokenize.NAME, 'struct',
1335     if token2.token_type == tokenize.SYNTAX and token2.name == '~':
1337     assert token.token_type == tokenize.NAME or token.name == '::', token
1339         tokenize.SYNTAX, '(')  # )
1365     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1386     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1392     if (token.token_type == tokenize.NAME and
1403     tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
1425     if tokens and isinstance(tokens[0], tokenize.Token):
1465     assert token.token_type == tokenize.SYNTAX, token
1470     if token.token_type == tokenize.NAME:
1478     tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
1503       assert token.token_type == tokenize.NAME, token
1522       assert next_token.token_type == tokenize.SYNTAX, next_token
1533     if class_token.token_type != tokenize.NAME:
1534       assert class_token.token_type == tokenize.SYNTAX, class_token
1540       if next_token.token_type == tokenize.NAME:
1547     if token.token_type == tokenize.SYNTAX:
1572     if token.token_type == tokenize.SYNTAX and token.name == '{':
1573       assert token.token_type == tokenize.SYNTAX, token
1582         if token.token_type != tokenize.NAME:
1583           assert token.token_type == tokenize.SYNTAX, token
1609     assert token.token_type == tokenize.SYNTAX, token
1611     internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
1629     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1658     assert token.token_type == tokenize.SYNTAX
1668     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1674     tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
1697     self._IgnoreUpTo(tokenize.SYNTAX, ';')
1700     self._IgnoreUpTo(tokenize.SYNTAX, ';')
1713   return AstBuilder(tokenize.GetTokens(source), filename)