Searched refs:IsPunctuation (Results 1 – 15 of 15) sorted by relevance
41 EXPECT_TRUE(unilib_->IsPunctuation('!')); in TEST_F()42 EXPECT_TRUE(unilib_->IsPunctuation('?')); in TEST_F()43 EXPECT_TRUE(unilib_->IsPunctuation('#')); in TEST_F()44 EXPECT_TRUE(unilib_->IsPunctuation('(')); in TEST_F()45 EXPECT_FALSE(unilib_->IsPunctuation('0')); in TEST_F()46 EXPECT_FALSE(unilib_->IsPunctuation('$')); in TEST_F()121 EXPECT_TRUE(unilib_->IsPunctuation(0x055E)); // ARMENIAN QUESTION MARK in TEST_F()122 EXPECT_TRUE(unilib_->IsPunctuation(0x066C)); // ARABIC THOUSANDS SEPARATOR in TEST_F()123 EXPECT_TRUE(unilib_->IsPunctuation(0x07F7)); // NKO SYMBOL GBAKURUNEN in TEST_F()124 EXPECT_TRUE(unilib_->IsPunctuation(0x10AF2)); // DOUBLE DOT WITHIN DOT in TEST_F()[all …]
32 bool IsPunctuation(char32 codepoint);
54 bool IsPunctuation(char32 codepoint) const;
64 bool UniLibBase::IsPunctuation(char32 codepoint) const { in IsPunctuation() function in libtextclassifier3::UniLibBase65 return libtextclassifier3::IsPunctuation(codepoint); in IsPunctuation()
569 bool IsPunctuation(char32 codepoint) { in IsPunctuation() function
67 bool IsPunctuation(wchar_t ch) { in IsPunctuation() function171 if (IsPunctuation(ch)) { in EncodeText()194 if (IsPunctuation(ch)) { in EncodeText()216 if (IsPunctuation(next)) { in EncodeText()222 if (IsPunctuation(ch)) { in EncodeText()228 if (IsPunctuation(ch)) { in EncodeText()
53 unilib.IsPunctuation(codepoint)) { in NormalizeTextCodepointWise()
51 is_whitespace || IsPunctuation(codepoint) || IsChineseLetter(codepoint); in TokenizeOnWhiteSpacePunctuationAndChineseLetter()
22 if (unilib_.IsPunctuation(*it)) { in GetSymbolType()
130 unilib_->IsPunctuation(*ending_begin_it)) { in TokensAreValidEnding()135 unilib_->IsPunctuation(*ending_begin_it) && in TokensAreValidEnding()
71 bool IsPunctuation(uint32_t word) { in IsPunctuation() function153 if (IsSpace(curWord) || IsPunctuation(curWord)) { in NeedDivision()159 if (IsSpace(prevWord) || IsPunctuation(prevWord)) { in NeedDivision()
50 if (IsPunctuation(r)) { in PreprocessToken()
160 if (unilib_->IsWhitespace(*it) || unilib_->IsPunctuation(*it)) { in FindIndexOfNextWhitespaceOrPunctuation()
296 (last_token.size() != 1 || !unilib_.IsPunctuation(last_token.at(0)))) { in ExecuteModel()
1412 "[\p{IsPunctuation}]" " <0>&</0>"1413 "[\P{IsPunctuation}]" "&<0> </0>"