Searched defs:max_bytes_per_token (Results 1 – 4 of 4) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | wordpiece_tokenizer.cc | 160 const absl::string_view token, const int max_bytes_per_token, in TokenizeL2RGreedy()
210 const absl::string_view token, const int max_bytes_per_token, in WordpieceTokenize()
235 const absl::string_view token, const int max_bytes_per_token, in WordpieceTokenize()
|
| D | bert_tokenizer.h | 59 int max_bytes_per_token = kDefaultMaxBytesPerToken; member
|
| /external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| D | bert_tokenizer_jni.cc | 34 JNIEnv* env, jobject thiz, jobject vocab_list, jint max_bytes_per_token, in Java_org_tensorflow_lite_support_text_tokenizers_BertTokenizer_nativeLoadResource()
|
| D | bert_tokenizer.h | 60 int max_bytes_per_token = kDefaultMaxBytesPerToken; member
|