Searched refs:tokenizer (Results 1 – 1 of 1) sorted by relevance
537 def __init__(self,tokenizer=None): argument538 self.tokenizer = tokenizer541 def setTokenizer(self,tokenizer): argument542 self.tokenizer = tokenizer545 self.tokenizer.nextToken(self.token)