Searched refs:unicode_text (Results 1 – 5 of 5) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | tokenizer.cc | 177 void Tokenizer::TokenizeSubstring(const UnicodeText& unicode_text, in TokenizeSubstring() argument
186 UnicodeText text = UnicodeText::Substring(unicode_text, span.first, in TokenizeSubstring()
202 void Tokenizer::InternalRetokenize(const UnicodeText& unicode_text, in InternalRetokenize() argument
224 TokenizeSubstring(unicode_text, span, &result); in InternalRetokenize()
229 TokenizeSubstring(unicode_text, span, &result); in InternalRetokenize()
|
| D | tokenizer.h | 112 void TokenizeSubstring(const UnicodeText& unicode_text, CodepointSpan span,
119 void InternalRetokenize(const UnicodeText& unicode_text,
|
| /external/libtextclassifier/native/annotator/ |
| D | feature-processor.h | 96 const UnicodeText& unicode_text, const CodepointSpan& span);
|
| D | feature-processor.cc | 366 const UnicodeText& unicode_text, const CodepointSpan& span) { in CodepointSpanToUnicodeTextRange() argument
367 auto begin = unicode_text.begin(); in CodepointSpanToUnicodeTextRange()
371 auto end = unicode_text.begin(); in CodepointSpanToUnicodeTextRange()
|
| /external/python/cpython2/Parser/ |
| D | tokenizer.c | 1711 PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace"); in dec_utf8() local
1712 if (unicode_text) { in dec_utf8()
1713 ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace"); in dec_utf8()
1714 Py_DECREF(unicode_text); in dec_utf8()
|