Searched refs:get_tokenizer (Results 1 – 5 of 5) sorted by relevance
13 def get_tokenizer(tokenizer_path): function
12 from executorch.extension.llm.tokenizer.utils import get_tokenizer72 self.tokenizer = get_tokenizer(tokenizer_path)
22 from executorch.extension.llm.tokenizer.utils import get_tokenizer169 tokenizer = get_tokenizer(args.tokenizer_path) # pyre-ignore
33 from executorch.extension.llm.tokenizer.utils import get_tokenizer235 tokenizer = get_tokenizer(tokenizer_path)
52 from executorch.extension.llm.tokenizer.utils import get_tokenizer71 sp_model = get_tokenizer(tokenizer_model_path)