93 inline size_t FindNextSpace(const char* input_ptr, size_t from, size_t length) {  in FindNextSpace()  argument
96     if (input_ptr[space_index] == kSpace) {  in FindNextSpace()
104 void SplitByCharInternal(std::vector<T>* tokens, const char* input_ptr,  in SplitByCharInternal()  argument
108         ::libtextclassifier3::GetNumBytesForUTF8Char(input_ptr + i);  in SplitByCharInternal()
110     tokens->emplace_back(input_ptr + i, bytes_read);  in SplitByCharInternal()
118 std::vector<std::string> SplitByChar(const char* input_ptr, size_t len,  in SplitByChar()  argument
121   SplitByCharInternal(&tokens, input_ptr, len, max_tokens);  in SplitByChar()
125 std::string ContractToken(const char* input_ptr, size_t len, size_t num_chars) {  in ContractToken()  argument
130     return input_ptr;  in ContractToken()
132   std::vector<std::string> char_tokens = SplitByChar(input_ptr, len, len);  in ContractToken()
161 void SplitBySpaceInternal(std::vector<T>* tokens, const char* input_ptr,  in SplitBySpaceInternal()  argument
167   while (start < last_index && input_ptr[start] == kSpace) {  in SplitBySpaceInternal()
170   auto end = FindNextSpace(input_ptr, start, last_index);  in SplitBySpaceInternal()
175       tokens->emplace_back(input_ptr + start, length);  in SplitBySpaceInternal()
179     end = FindNextSpace(input_ptr, start, last_index);  in SplitBySpaceInternal()
183     tokens->emplace_back(input_ptr + start, length);  in SplitBySpaceInternal()
187 std::vector<std::string> SplitBySpace(const char* input_ptr, size_t len,  in SplitBySpace()  argument
190   SplitBySpaceInternal(&tokens, input_ptr, len, max_input, max_tokens);  in SplitBySpace()
211   std::string Normalize(const char* input_ptr, size_t len,  in Normalize()  argument
213     std::string normalized(input_ptr, std::min(len, max_input));  in Normalize()
246   std::string NormalizeInternal(const char* input_ptr, size_t len) {  in NormalizeInternal()  argument
250       char c = input_ptr[i];  in NormalizeInternal()
253         if (i > 0 && input_ptr[i - 1] != ' ' && normalized.back() != ' ') {  in NormalizeInternal()
259         if (i + 1 < len && input_ptr[i + 1] != ' ' && c != '\'') {  in NormalizeInternal()
283   std::vector<std::string> Tokenize(const char* input_ptr, size_t len,  in Tokenize()  argument
287       return SplitBySpace(input_ptr, len, max_input, max_tokens);  in Tokenize()
295     while (start < last_index && input_ptr[start] == kSpace) {  in Tokenize()
298     auto end = FindNextSeparator(input_ptr, start, last_index);  in Tokenize()
303       if (length > 0) tokens.emplace_back(input_ptr + start, length);  in Tokenize()
306       char separator = input_ptr[end];  in Tokenize()
308         tokens.emplace_back(input_ptr + end, 1);  in Tokenize()
312       end = FindNextSeparator(input_ptr, end + 1, last_index);  in Tokenize()
315     if (length > 0) tokens.emplace_back(input_ptr + start, length);  in Tokenize()
330   size_t FindNextSeparator(const char* input_ptr, size_t from,  in FindNextSeparator()  argument
334       char c = input_ptr[index];  in FindNextSeparator()
337         if (index + 1 < length && is_numeric(input_ptr[index + 1])) {  in FindNextSeparator()
338           c = input_ptr[++index];  in FindNextSeparator()