32   TokenFeatureExtractorOptions options;  in TEST()  local
33   options.num_buckets = 1000;  in TEST()
34   options.chargram_orders = std::vector<int>{1, 2, 3};  in TEST()
35   options.extract_case_feature = true;  in TEST()
36   options.unicode_aware_features = false;  in TEST()
37   options.extract_selection_mask_feature = true;  in TEST()
38   TestingTokenFeatureExtractor extractor(options);  in TEST()
102   TokenFeatureExtractorOptions options;  in TEST()  local
103   options.num_buckets = 1000;  in TEST()
104   options.chargram_orders = std::vector<int>{1, 2, 3};  in TEST()
105   options.extract_case_feature = true;  in TEST()
106   options.unicode_aware_features = true;  in TEST()
107   options.extract_selection_mask_feature = true;  in TEST()
108   TestingTokenFeatureExtractor extractor(options);  in TEST()
172   TokenFeatureExtractorOptions options;  in TEST()  local
173   options.num_buckets = 1000;  in TEST()
174   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
175   options.extract_case_feature = true;  in TEST()
176   options.unicode_aware_features = true;  in TEST()
177   options.extract_selection_mask_feature = false;  in TEST()
178   TokenFeatureExtractor extractor(options);  in TEST()
206   TokenFeatureExtractorOptions options;  in TEST()  local
207   options.num_buckets = 1000;  in TEST()
208   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
209   options.remap_digits = true;  in TEST()
210   options.unicode_aware_features = false;  in TEST()
211   TokenFeatureExtractor extractor(options);  in TEST()
230   TokenFeatureExtractorOptions options;  in TEST()  local
231   options.num_buckets = 1000;  in TEST()
232   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
233   options.remap_digits = true;  in TEST()
234   options.unicode_aware_features = true;  in TEST()
235   TokenFeatureExtractor extractor(options);  in TEST()
254   TokenFeatureExtractorOptions options;  in TEST()  local
255   options.num_buckets = 1000;  in TEST()
256   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
257   options.lowercase_tokens = true;  in TEST()
258   options.unicode_aware_features = false;  in TEST()
259   TokenFeatureExtractor extractor(options);  in TEST()
277   TokenFeatureExtractorOptions options;  in TEST()  local
278   options.num_buckets = 1000;  in TEST()
279   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
280   options.lowercase_tokens = true;  in TEST()
281   options.unicode_aware_features = true;  in TEST()
282   TokenFeatureExtractor extractor(options);  in TEST()
295   TokenFeatureExtractorOptions options;  in TEST()  local
296   options.num_buckets = 1000;  in TEST()
297   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
298   options.remap_digits = false;  in TEST()
299   options.unicode_aware_features = false;  in TEST()
300   options.regexp_features.push_back("^[a-z]+$");  // all lower case.  in TEST()
301   options.regexp_features.push_back("^[0-9]+$");  // all digits.  in TEST()
302   TokenFeatureExtractor extractor(options);  in TEST()
327   TokenFeatureExtractorOptions options;  in TEST()  local
328   options.num_buckets = 1000;  in TEST()
329   options.chargram_orders = std::vector<int>{22};  in TEST()
330   options.extract_case_feature = true;  in TEST()
331   options.unicode_aware_features = true;  in TEST()
332   options.extract_selection_mask_feature = true;  in TEST()
333   TestingTokenFeatureExtractor extractor(options);  in TEST()
351   TokenFeatureExtractorOptions options;  in TEST()  local
352   options.num_buckets = 1000;  in TEST()
353   options.chargram_orders = std::vector<int>{1, 2, 3, 4, 5};  in TEST()
354   options.extract_case_feature = true;  in TEST()
355   options.unicode_aware_features = true;  in TEST()
356   options.extract_selection_mask_feature = true;  in TEST()
357   TestingTokenFeatureExtractor extractor_unicode(options);  in TEST()
359   options.unicode_aware_features = false;  in TEST()
360   TestingTokenFeatureExtractor extractor_ascii(options);  in TEST()
384   TokenFeatureExtractorOptions options;  in TEST()  local
385   options.num_buckets = 1000;  in TEST()
386   options.chargram_orders = std::vector<int>{1, 2};  in TEST()
387   options.extract_case_feature = true;  in TEST()
388   options.unicode_aware_features = false;  in TEST()
389   options.extract_selection_mask_feature = true;  in TEST()
391   TestingTokenFeatureExtractor extractor(options);  in TEST()