8  *      http://www.apache.org/licenses/LICENSE-2.0
17 #include "smartselect/feature-processor.h"
35   // clang-format off  in TEST()
42   // clang-format on  in TEST()
52   // clang-format off  in TEST()
58   // clang-format on  in TEST()
68   // clang-format off  in TEST()
74   // clang-format on  in TEST()
84   // clang-format off  in TEST()
89   // clang-format on  in TEST()
99   // clang-format off  in TEST()
106   // clang-format on  in TEST()
112   // clang-format off  in TEST()
119   // clang-format on  in TEST()
130   // clang-format off  in TEST()
137   // clang-format on  in TEST()
148   // clang-format off  in TEST()
155   // clang-format on  in TEST()
166   // clang-format off  in TEST()
173   // clang-format on  in TEST()
184   // clang-format off  in TEST()
191   // clang-format on  in TEST()
219   config->set_start(32);  in TEST()
220   config->set_end(33);  in TEST()
221   config->set_role(TokenizationCodepointRange::WHITESPACE_SEPARATOR);  in TEST()
225   ASSERT_EQ(3, tokens.size());  in TEST()
298   // Selection of length 3. Exactly aligned indices.  in TEST()
310   EXPECT_EQ(token_index, 3);  in TEST()
312   // Selection marks sub-token range, with no tokens in it.  in TEST()
319   // Selection of length 2. Sub-token indices.  in TEST()
321       {3, 25},  in TEST()
326   // Selection of length 1. Sub-token indices.  in TEST()
335   EXPECT_EQ(token_index, -1);  in TEST()
346   config->set_start(32);  in TEST()
347   config->set_end(33);  in TEST()
348   config->set_role(TokenizationCodepointRange::WHITESPACE_SEPARATOR);  in TEST()
352   range->set_start(0);  in TEST()
353   range->set_end(128);  in TEST()
356   range->set_start(10000);  in TEST()
357   range->set_end(10001);  in TEST()
360   range->set_start(20000);  in TEST()
361   range->set_end(30000);  in TEST()
369               FloatEq(2.0 / 3));  in TEST()
374       -1, feature_processor.supported_codepoint_ranges_));  in TEST()
422       Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0),  Token("3", 0, 0),  in TEST()
434   // clang-format off  in TEST()
440   // clang-format on  in TEST()
447   // clang-format off  in TEST()
451                                         Token("3", 0, 0),  in TEST()
453   // clang-format on  in TEST()
460   // clang-format off  in TEST()
466   // clang-format on  in TEST()
472       Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0),  Token("3", 0, 0),  in TEST()
484   internal::StripOrPadTokens({2, 3}, 2, &tokens, &click_index);  in TEST()
485   // clang-format off  in TEST()
491                                         Token("3", 0, 0),  in TEST()
494   // clang-format on  in TEST()
500   internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);  in TEST()
501   // clang-format off  in TEST()
504                                         Token("3", 0, 0),  in TEST()
511   // clang-format on  in TEST()
517   internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);  in TEST()
518   // clang-format off  in TEST()
528   // clang-format on  in TEST()
540             // clang-format off  in TEST()
546   // clang-format on  in TEST()
559             // clang-format off  in TEST()
569   // clang-format on  in TEST()
579   config->set_start(32);  in TEST()
580   config->set_end(33);  in TEST()
581   config->set_role(TokenizationCodepointRange::WHITESPACE_SEPARATOR);  in TEST()
585   range->set_start(0);  in TEST()
586   range->set_end(128);  in TEST()
589   range->set_start(128);  in TEST()
590   range->set_end(256);  in TEST()
593   range->set_start(256);  in TEST()
594   range->set_end(384);  in TEST()
597   range->set_start(384);  in TEST()
598   range->set_end(592);  in TEST()
602       "こんにちはJapanese-ląnguagę text 世界 http://www.google.com/");  in TEST()
604             // clang-format off  in TEST()
606                                 Token("Japanese-ląnguagę", 5, 22),  in TEST()
610   // clang-format on  in TEST()