3 tokenize(readline) is a generator that breaks a stream of bytes into
7 It accepts a readline-like method which is called repeatedly to get the
326         t1 = [tok[:2] for tok in tokenize(f.readline)]
328         readline = BytesIO(newcode).readline
329         t2 = [tok[:2] for tok in tokenize(readline)]
350 def detect_encoding(readline):  argument
353     be used to decode a Python source file.  It requires one argument, readline,
356     It will call readline a maximum of twice, and return the encoding used
368         filename = readline.__self__.name
376             return readline()
449         encoding, lines = detect_encoding(buffer.readline)
458 def tokenize(readline):  argument
460     The tokenize() generator requires one argument, readline, which
462     readline() method of built-in file objects.  Each call to the function
463     should return one line of input as bytes.  Alternatively, readline
465         readline = open(myfile, 'rb').__next__  # Example of alternate readline
477     encoding, consumed = detect_encoding(readline)
478     rl_gen = _itertools.chain(consumed, iter(readline, b""))
486 def generate_tokens(readline):  argument
489     This has the same API as tokenize(), except that it expects the *readline*
492     return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True)
526                 tokens = list(tokenize(f.readline))
530                 sys.stdin.readline, extra_tokens=True)