Lines Matching refs:TokenInfo
45 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): class
440 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
465 yield TokenInfo(STRING, contstr + line[:end],
470 yield TokenInfo(ERRORTOKEN, contstr + line,
499 yield TokenInfo(COMMENT, comment_token,
503 yield TokenInfo(NL, line[pos:],
509 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
517 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
535 yield TokenInfo(NUMBER, token, spos, epos, line)
538 yield TokenInfo(NL, token, spos, epos, line)
540 yield TokenInfo(NEWLINE, token, spos, epos, line)
544 yield TokenInfo(COMMENT, token, spos, epos, line)
552 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
587 yield TokenInfo(STRING, token, spos, epos, line)
590 yield TokenInfo(NAME, token, spos, epos, line)
598 yield TokenInfo(OP, token, spos, epos, line)
600 yield TokenInfo(ERRORTOKEN, line[pos],
606 yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
608 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
609 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')