Lines Matching refs:end_offset
96 std::vector<int>* end_offset, int* num_word_pieces) { in NoTokenFound() argument
100 end_offset->push_back(token.length()); in NoTokenFound()
103 end_offset->push_back(token.length()); in NoTokenFound()
115 std::vector<int>* end_offset) { in AddWord() argument
126 end_offset->push_back(byte_end); in AddWord()
137 std::vector<int>* end_offset) { in AddUnknownCharacter() argument
139 end_offset->push_back(byte_end); in AddUnknownCharacter()
165 std::vector<int>* end_offset, int* num_word_pieces) { in TokenizeL2RGreedy() argument
193 begin_offset, end_offset, num_word_pieces); in TokenizeL2RGreedy()
201 end_offset->insert(end_offset->end(), candidate_end_offsets.begin(), in TokenizeL2RGreedy()
215 std::vector<int>* end_offset, int* num_word_pieces) { in WordpieceTokenize() argument
221 end_offset->push_back(unknown_token.size()); in WordpieceTokenize()
225 end_offset->push_back(token.size()); in WordpieceTokenize()
232 begin_offset, end_offset, num_word_pieces); in WordpieceTokenize()
240 std::vector<int>* end_offset, int* num_word_pieces) { in WordpieceTokenize() argument
245 subwords, begin_offset, end_offset, num_word_pieces); in WordpieceTokenize()