Searched refs:token_unicode (Results 1 – 2 of 2) sorted by relevance
35 UnicodeText token_unicode = UTF8ToUnicodeText(value.data(), value.size(), in AppendTokenSymbols() local38 auto token_end = token_unicode.end(); in AppendTokenSymbols()39 auto it = token_unicode.begin(); in AppendTokenSymbols()
110 UnicodeText token_unicode = in ExtractDenseFeatures() local112 if (!token.value.empty() && unilib_.IsUpper(*token_unicode.begin())) { in ExtractDenseFeatures()140 UnicodeText token_unicode = in ExtractDenseFeatures() local147 auto matcher = regex_patterns_[i]->Matcher(token_unicode); in ExtractDenseFeatures()