Home
last modified time | relevance | path

Searched refs:UnicodeText (Results 1 – 8 of 8) sorted by relevance

/external/libtextclassifier/util/utf8/
Dunicodetext.cc27 void UnicodeText::Repr::PointTo(const char* data, int size) { in PointTo()
35 void UnicodeText::Repr::Copy(const char* data, int size) { in Copy()
40 void UnicodeText::Repr::resize(int new_size) { in resize()
52 void UnicodeText::Repr::reserve(int new_capacity) { in reserve()
70 void UnicodeText::Repr::append(const char* bytes, int byte_length) { in append()
76 void UnicodeText::Repr::clear() { in clear()
85 UnicodeText::UnicodeText() {} in UnicodeText() function in libtextclassifier::UnicodeText
87 UnicodeText::UnicodeText(const UnicodeText& src) { Copy(src); } in UnicodeText() function in libtextclassifier::UnicodeText
89 UnicodeText& UnicodeText::Copy(const UnicodeText& src) { in Copy()
94 UnicodeText& UnicodeText::PointToUTF8(const char* buffer, int byte_length) { in PointToUTF8()
[all …]
Dunicodetext.h63 class UnicodeText {
67 UnicodeText(); // Create an empty text.
68 UnicodeText(const UnicodeText& src);
69 ~UnicodeText();
132 friend class UnicodeText;
144 UnicodeText& Copy(const UnicodeText& src);
145 UnicodeText& PointToUTF8(const char* utf8_buffer, int byte_length);
146 UnicodeText& CopyUTF8(const char* utf8_buffer, int byte_length);
149 UnicodeText& AppendUTF8(const char* utf8, int len);
187 typedef std::pair<UnicodeText::const_iterator, UnicodeText::const_iterator>
[all …]
/external/libtextclassifier/smartselect/
Dtoken-feature-extractor.cc52 UnicodeText* remapped) { in RemapTokenUnicode()
58 UnicodeText word = UTF8ToUnicodeText(token, /*do_copy=*/false); in RemapTokenUnicode()
155 UnicodeText word = UTF8ToUnicodeText(token.value, /*do_copy=*/false); in ExtractCharactergramFeaturesUnicode()
183 const UnicodeText feature_word_unicode = in ExtractCharactergramFeaturesUnicode()
191 UnicodeText::const_iterator it_start = feature_word_unicode.begin(); in ExtractCharactergramFeaturesUnicode()
192 UnicodeText::const_iterator it_end = feature_word_unicode.end(); in ExtractCharactergramFeaturesUnicode()
198 UnicodeText::const_iterator it_chargram_start = it_start; in ExtractCharactergramFeaturesUnicode()
199 UnicodeText::const_iterator it_chargram_end = it_start; in ExtractCharactergramFeaturesUnicode()
235 UnicodeText token_unicode = in Extract()
Dfeature-processor.cc67 const UnicodeText token_word = in SplitTokensOnSelectionBoundaries()
72 std::vector<UnicodeText::const_iterator> split_points; in SplitTokensOnSelectionBoundaries()
116 void FindSubstrings(const UnicodeText& t, const std::set<char32>& codepoints, in FindSubstrings()
118 UnicodeText::const_iterator start = t.begin(); in FindSubstrings()
119 UnicodeText::const_iterator curr = start; in FindSubstrings()
120 UnicodeText::const_iterator end = t.end(); in FindSubstrings()
137 const UnicodeText context_unicode = UTF8ToUnicodeText(context, in StripTokensFromOtherLines()
464 const UnicodeText value = in SupportedCodepointsRatio()
706 const UnicodeText unicode_text = in InternalRetokenize()
712 const UnicodeText unicode_token_value = in InternalRetokenize()
[all …]
Dtext-classification-model.cc49 const UnicodeText unicode_str = UTF8ToUnicodeText(str, /*do_copy=*/false); in CountDigits()
63 UnicodeText context_unicode = UTF8ToUnicodeText(context, /*do_copy=*/false); in StripPunctuation()
74 UnicodeText::const_iterator it_from_begin = context_unicode.begin(); in StripPunctuation()
83 UnicodeText::const_iterator it_from_end = context_unicode.begin(); in StripPunctuation()
286 const UnicodeText context_unicode = in SuggestSelection()
Dtokenizer.cc68 UnicodeText context_unicode = UTF8ToUnicodeText(utf8_text, /*do_copy=*/false); in Tokenize()
Dfeature-processor.h227 void TokenizeSubstring(const UnicodeText& unicode_text, CodepointSpan span,
/external/libtextclassifier/
Dtextclassifier_jni.cc102 const libtextclassifier::UnicodeText unicode_str = in ConvertIndicesBMPUTF8()