Lines Matching refs:tokenize

165 import tokenize
375                 if ttype == tokenize.STRING:
378                 elif ttype not in (tokenize.COMMENT, tokenize.NL):
382             if ttype == tokenize.NAME and tstring in ('class', 'def'):
385         if ttype == tokenize.NAME and tstring in opts.keywords:
390         if ttype == tokenize.OP and tstring == ':':
395         if ttype == tokenize.STRING:
398         elif ttype not in (tokenize.NEWLINE, tokenize.INDENT,
399                            tokenize.COMMENT):
404         if ttype == tokenize.OP and tstring == '(':
412         if ttype == tokenize.OP and tstring == ')':
420         elif ttype == tokenize.STRING:
422         elif ttype not in [tokenize.COMMENT, token.INDENT, token.DEDENT,
423                            token.NEWLINE, tokenize.NL]:
638                 tokenize.tokenize(fp.readline, eater)
639             except tokenize.TokenError, e: