Searched refs:_generate_tokens_from_c_tokenizer (Results 1 – 1 of 1) sorted by relevance
484 yield from _generate_tokens_from_c_tokenizer(rl_gen.__next__, encoding, extra_tokens=True)492 return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True)529 tokens = _generate_tokens_from_c_tokenizer(567 def _generate_tokens_from_c_tokenizer(source, encoding=None, extra_tokens=False): function