Home
last modified time | relevance | path

Searched refs:i18n_utils (Results 1 – 12 of 12) sorted by relevance

/external/icing/icing/util/
Di18n-utils_test.cc37 EXPECT_EQ(i18n_utils::IsPunctuationAt(ascii_string, /*position=*/0), in TEST()
46 EXPECT_TRUE(i18n_utils::IsAlphabeticAt(kSomeAscii, /*position=*/0)); // 'i' in TEST()
47 EXPECT_TRUE(i18n_utils::IsAlphabeticAt(kSomeAscii, /*position=*/1)); // 'J' in TEST()
48 EXPECT_FALSE(i18n_utils::IsAlphabeticAt(kSomeAscii, /*position=*/2)); // '?' in TEST()
49 EXPECT_FALSE(i18n_utils::IsAlphabeticAt(kSomeAscii, /*position=*/3)); // '9' in TEST()
53 i18n_utils::IsAlphabeticAt(kSomeNonAscii, /*position=*/0)); // '��' in TEST()
54 EXPECT_EQ(i18n_utils::GetUtf8Length(i18n_utils::GetUChar32At( in TEST()
58 i18n_utils::IsAlphabeticAt(kSomeNonAscii, /*position=*/4)); // 'ñ' in TEST()
59 EXPECT_EQ(i18n_utils::GetUtf8Length(i18n_utils::GetUChar32At( in TEST()
63 i18n_utils::IsAlphabeticAt(kSomeNonAscii, /*position=*/6)); // '①' in TEST()
[all …]
Dcharacter-iterator.cc25 while (!i18n_utils::IsLeadUtf8Byte(text[current_byte_index])) { in GetUTF8StartPosition()
46 i18n_utils::GetUChar32At(text_.data(), text_.length(), utf8_index_); in AdvanceToUtf8()
47 if (uchar32 == i18n_utils::kInvalidUChar32) { in AdvanceToUtf8()
51 int utf8_length = i18n_utils::GetUtf8Length(uchar32); in AdvanceToUtf8()
57 utf16_index_ += i18n_utils::GetUtf16Length(uchar32); in AdvanceToUtf8()
79 i18n_utils::GetUChar32At(text_.data(), text_.length(), utf8_index_); in RewindToUtf8()
80 if (uchar32 == i18n_utils::kInvalidUChar32) { in RewindToUtf8()
84 utf16_index_ -= i18n_utils::GetUtf16Length(uchar32); in RewindToUtf8()
99 i18n_utils::GetUChar32At(text_.data(), text_.length(), utf8_index_); in AdvanceToUtf16()
100 if (uchar32 == i18n_utils::kInvalidUChar32) { in AdvanceToUtf16()
[all …]
Di18n-utils.h29 namespace i18n_utils {
Di18n-utils.cc33 namespace i18n_utils { namespace
/external/icing/icing/transform/map/
Dmap-normalizer.cc38 if (i18n_utils::IsAscii(term[i])) { in NormalizeTerm()
41 } else if (i18n_utils::IsLeadUtf8Byte(term[i])) { in NormalizeTerm()
42 UChar32 uchar32 = i18n_utils::GetUChar32At(term.data(), term.length(), i); in NormalizeTerm()
43 if (uchar32 == i18n_utils::kInvalidUChar32) { in NormalizeTerm()
48 int utf8_length = i18n_utils::GetUtf8Length(uchar32); in NormalizeTerm()
49 if (i18n_utils::GetUtf16Length(uchar32) > 1) { in NormalizeTerm()
63 if (i18n_utils::IsAscii(iterator->second)) { in NormalizeTerm()
69 i18n_utils::AppendUchar32ToUtf8(&normalized_text, iterator->second); in NormalizeTerm()
79 i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); in NormalizeTerm()
/external/icing/icing/transform/icu/
Dicu-normalizer.cc69 if (i18n_utils::IsAscii(uchar32_in)) { in DiacriticCharToAscii()
90 !U_FAILURE(status) && i18n_utils::IsAscii(decomposition_buffer[0])) { in DiacriticCharToAscii()
140 i18n_utils::GetUChar32At(term.data(), term.length(), 0); in NormalizeTerm()
141 if (normalizer2 != nullptr && first_uchar32 != i18n_utils::kInvalidUChar32 && in NormalizeTerm()
150 i18n_utils::SafeTruncateUtf8(&normalized_text, max_term_byte_size_); in NormalizeTerm()
161 if (i18n_utils::IsAscii(term[i])) { in NormalizeLatin()
163 } else if (i18n_utils::IsLeadUtf8Byte(term[i])) { in NormalizeLatin()
164 UChar32 uchar32 = i18n_utils::GetUChar32At(term.data(), term.length(), i); in NormalizeLatin()
165 if (uchar32 == i18n_utils::kInvalidUChar32) { in NormalizeLatin()
179 int utf8_length = i18n_utils::GetUtf8Length(uchar32); in NormalizeLatin()
[all …]
/external/icing/icing/tokenization/
Dplain-tokenizer.cc43 return !i18n_utils::IsWhitespaceAt(term, /*position=*/0) && in IsValidTerm()
44 !i18n_utils::IsPunctuationAt(term, /*position=*/0); in IsValidTerm()
Draw-query-tokenizer.cc362 if (!i18n_utils::IsAscii(text[pos])) { in GetContentTerm()
545 if (!i18n_utils::IsAscii(term[0])) { in ProcessTerm()
615 } else if (!i18n_utils::IsAscii(term[0])) { in ProcessTerms()
/external/icing/icing/tokenization/reverse_jni/
Dreverse-jni-language-segmenter.cc290 if (i18n_utils::IsAscii(text_[term_start_.utf8_index()])) { in IsValidTerm()
296 if (i18n_utils::IsAlphabeticAt(text_, term_start_.utf8_index())) { in IsValidTerm()
/external/icing/icing/tokenization/icu/
Dicu-language-segmenter.cc297 if (i18n_utils::IsAscii(text_[term_start_index_])) { in IsValidSegment()
301 UChar32 uchar32 = i18n_utils::GetUChar32At(text_.data(), text_.length(), in IsValidSegment()
/external/icing/icing/legacy/index/
Dicing-dynamic-trie.cc1967 if (!i18n_utils::IsAscii(cur_[cur_len_ - 1])) { in LeftBranchToUtf8End()
1976 UChar32 uchar32 = i18n_utils::GetUChar32At(cur_, cur_len_, 0); in LeftBranchToUtf8End()
1977 if (uchar32 == i18n_utils::kInvalidUChar32 && in LeftBranchToUtf8End()
2173 if (utf8 && i18n_utils::IsLeadUtf8Byte(*cur_key)) { in FindBestNode()
2215 size_t offset = i18n_utils::SafeTruncateUtf8Length(key, cur_key - key); in FindNewBranchingPrefixLength()
2246 len = i18n_utils::SafeTruncateUtf8Length(key, len); in FindBranchingPrefixLengths()
/external/icing/icing/result/
Dsnippet-retriever.cc184 if (!i18n_utils::IsPunctuationAt(value, window_end_exclusive.utf8_index(), in IncludeTrailingPunctuation()