Searched defs:subwords (Results 1 – 6 of 6) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | wordpiece_tokenizer.cc | 94 std::vector<std::string>* subwords, in NoTokenFound()
114 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in AddWord()
135 std::vector<std::string>* subwords, in AddUnknownCharacter()
164 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in TokenizeL2RGreedy()
214 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in WordpieceTokenize()
239 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in WordpieceTokenize()
|
| D | bert_tokenizer.cc | 75 std::vector<std::string>& subwords = result.subwords; in Tokenize() local
|
| /external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| D | tokenizer.h | 31 std::vector<std::string> subwords; member
|
| D | sentencepiece_tokenizer.h | 48 std::vector<std::string>& subwords = result.subwords; in Tokenize() local
|
| D | tokenizer_jni_lib.cc | 39 std::vector<std::string> subwords = tokenize_result.subwords; in nativeTokenize() local
|
| D | bert_tokenizer.cc | 65 std::vector<std::string>& subwords = result.subwords; in TokenizeWordpiece() local
|