Searched refs:internal_tokenizer_codepoint_ranges (Results 1 – 9 of 9) sorted by relevance
| /external/libtextclassifier/native/annotator/grammar/ |
| D | utils.cc | 39 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer()
42 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer()
43 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
|
| /external/libtextclassifier/native/utils/ |
| D | tokenizer.h | 56 internal_tokenizer_codepoint_ranges,
65 internal_tokenizer_codepoint_ranges, in Tokenizer() argument
69 internal_tokenizer_codepoint_ranges, split_on_script_change, in Tokenizer()
|
| D | tokenizer.cc | 32 internal_tokenizer_codepoint_ranges, in Tokenizer() argument
51 SortCodepointRanges(internal_tokenizer_codepoint_ranges, in Tokenizer()
|
| D | tokenizer.fbs | 29 // consisting only of codepoints in internal_tokenizer_codepoint_ranges
|
| D | tokenizer_test.cc | 35 internal_tokenizer_codepoint_ranges, in TestingTokenizer() argument
40 internal_tokenizer_codepoint_ranges, split_on_script_change, in TestingTokenizer()
|
| /external/libtextclassifier/native/actions/ |
| D | feature-processor.cc | 54 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in CreateTokenizer()
57 options->internal_tokenizer_codepoint_ranges()->begin(), in CreateTokenizer()
58 options->internal_tokenizer_codepoint_ranges()->end()); in CreateTokenizer()
|
| D | actions_model.fbs | 139 internal_tokenizer_codepoint_ranges:[CodepointRange];
|
| /external/libtextclassifier/native/annotator/ |
| D | feature-processor.cc | 40 if (options->internal_tokenizer_codepoint_ranges() != nullptr) { in BuildTokenizer()
43 options->internal_tokenizer_codepoint_ranges()->begin(), in BuildTokenizer()
44 options->internal_tokenizer_codepoint_ranges()->end()); in BuildTokenizer()
|
| D | model.fbs | 369 internal_tokenizer_codepoint_ranges:[CodepointRange];
742 internal_tokenizer_codepoint_ranges:[CodepointRange];
|