Lines Matching refs:TokenInfo
46 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): class
442 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
467 yield TokenInfo(STRING, contstr + line[:end],
472 yield TokenInfo(ERRORTOKEN, contstr + line,
501 yield TokenInfo(COMMENT, comment_token,
505 yield TokenInfo(NL, line[pos:],
511 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
519 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
537 yield TokenInfo(NUMBER, token, spos, epos, line)
540 yield TokenInfo(NL, token, spos, epos, line)
542 yield TokenInfo(NEWLINE, token, spos, epos, line)
546 yield TokenInfo(COMMENT, token, spos, epos, line)
554 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
589 yield TokenInfo(STRING, token, spos, epos, line)
592 yield TokenInfo(NAME, token, spos, epos, line)
600 yield TokenInfo(OP, token, spos, epos, line)
602 yield TokenInfo(ERRORTOKEN, line[pos],
608 yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
610 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
611 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')