Searched refs:vocab_map (Results 1 – 2 of 2) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | wordpiece_tokenizer.cc | 30 const WordpieceVocab* vocab_map, bool* in_vocab) { in Lookup() argument
40 return vocab_map->Contains(lookup_value, in_vocab); in Lookup()
51 bool split_unknown_characters, const WordpieceVocab* vocab_map, in LongestMatchStartingAt() argument
73 vocab_map, &in_vocab); in LongestMatchStartingAt()
163 bool split_unknown_characters, const WordpieceVocab* vocab_map, in TokenizeL2RGreedy() argument
176 split_unknown_characters, vocab_map, &byte_end, &found_subword, in TokenizeL2RGreedy()
213 bool split_unknown_characters, const WordpieceVocab* vocab_map, in WordpieceTokenize() argument
230 split_unknown_characters, vocab_map, subwords, in WordpieceTokenize()
237 const std::string& unknown_token, const WordpieceVocab* vocab_map, in WordpieceTokenize() argument
243 /* split_unknown_characters= */ false, vocab_map, in WordpieceTokenize()
|
| D | wordpiece_tokenizer.h | 48 bool split_unknown_characters, const WordpieceVocab* vocab_map,
57 const std::string& unknown_token, const WordpieceVocab* vocab_map,
|