Searched defs:byte_start (Results 1 – 1 of 1) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | wordpiece_tokenizer.cc | 28 LookupStatus Lookup(int byte_start, int byte_end, const absl::string_view token, in Lookup()
49 int byte_start, const absl::string_view token, in LongestMatchStartingAt()
112 void AddWord(const absl::string_view token, int byte_start, int byte_end, in AddWord()
131 void AddUnknownCharacter(const absl::string_view token, int byte_start, in AddUnknownCharacter()
170 for (int byte_start = 0; byte_start < token_len;) { in TokenizeL2RGreedy() local
|