Searched defs:token_unicode (Results 1 – 3 of 3) sorted by relevance
89 const UnicodeText token_unicode = UTF8ToUnicodeText(token, /*do_copy=*/false); in TokenizeSingleToken() local111 const UnicodeText token_unicode = in TokenizeIntoWordpieces() local
110 UnicodeText token_unicode = in ExtractDenseFeatures() local140 UnicodeText token_unicode = in ExtractDenseFeatures() local
35 UnicodeText token_unicode = UTF8ToUnicodeText(value.data(), value.size(), in AppendTokenSymbols() local