Searched refs:sp_model (Results 1 – 6 of 6) sorted by relevance
26 self.sp_model = SentencePieceProcessor(model_file=model_path)30 self.n_words: int = self.sp_model.vocab_size()31 self.bos_id: int = self.sp_model.bos_id()32 self.eos_id: int = self.sp_model.eos_id()37 assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()42 t = self.sp_model.encode(s)51 return self.sp_model.decode(t)55 return self.sp_model.decode(t)88 t = self.sp_model.id_to_piece(i)90 s = self.sp_model.get_score(i)
210 self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))229 return len(self.sp_model.encode(str(self.unk_token)))239 sp_model = f.read()243 model = model_pb2.ModelProto.FromString(sp_model)247 sp_model = model.SerializeToString()248 tokenizer.LoadFromSerializedProto(sp_model)254 state["sp_model_proto"] = self.sp_model.serialized_model_proto()259 self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)260 self.sp_model.LoadFromSerializedProto(self.sp_model_proto)265 return self.sp_model.get_piece_size()[all …]
89 absl::string_view sp_model = in Java_org_tensorflow_lite_task_text_qa_BertQuestionAnswerer_initJniWithAlbertByteBuffers() local94 model.data(), model.size(), sp_model.data(), sp_model.size()); in Java_org_tensorflow_lite_task_text_qa_BertQuestionAnswerer_initJniWithAlbertByteBuffers()
227 sp_model = _GetSentencepieceModel()235 tftext_sp = tensorflow_text.SentencepieceTokenizer(sp_model)236 opt_sp = sentencepiece_tokenizer.SentencepieceTokenizer(sp_model)
186 sp_model = SentencePieceProcessor(model_file=tokenizer_model_path)191 token_list = [sp_model.bos_id()]193 token_list += sp_model.encode(prompt)205 while token_list[-1] != sp_model.eos_id() and pos < 128:
71 sp_model = get_tokenizer(tokenizer_model_path)76 token_list = sp_model.encode(user_prompts, bos=True, eos=False)79 while token_list[-1] != sp_model.eos_id and pos < 511: