Lines Matching refs:tokenize

165 import tokenize
369                 if ttype == tokenize.STRING:
372                 elif ttype not in (tokenize.COMMENT, tokenize.NL):
376             if ttype == tokenize.NAME and tstring in ('class', 'def'):
379         if ttype == tokenize.NAME and tstring in opts.keywords:
384         if ttype == tokenize.OP and tstring == ':':
389         if ttype == tokenize.STRING:
392         elif ttype not in (tokenize.NEWLINE, tokenize.INDENT,
393                            tokenize.COMMENT):
398         if ttype == tokenize.OP and tstring == '(':
406         if ttype == tokenize.OP and tstring == ')':
414         elif ttype == tokenize.STRING:
416         elif ttype not in [tokenize.COMMENT, token.INDENT, token.DEDENT,
417                            token.NEWLINE, tokenize.NL]:
632                 tokenize.tokenize(fp.readline, eater)
633             except tokenize.TokenError, e: