Searched refs:TokenizerUtilTest (Results 1 – 1 of 1) sorted by relevance
25 TEST(TokenizerUtilTest, TokenizeOnSpace) { in TEST() argument68 TEST(TokenizerUtilTest, TokenizeOnDelimiters) { in TEST() argument99 TEST(TokenizerUtilTest, TokenizeOnDelimitersKeepNoSpace) { in TEST() argument143 TEST(TokenizerUtilTest, SimpleEnglishWithPunctuation) { in TEST() argument155 TEST(TokenizerUtilTest, InputDoesNotEndWithDelimiter) { in TEST() argument166 TEST(TokenizerUtilTest, OnlySpace) { in TEST() argument175 TEST(TokenizerUtilTest, Punctuation) { in TEST() argument189 TEST(TokenizerUtilTest, ChineseCharacters) { in TEST() argument