Searched refs:split_on_script_change (Results 1 – 3 of 3) sorted by relevance
| /external/libtextclassifier/native/utils/ |
| D | tokenizer.h | 57 const bool split_on_script_change,
66 const bool split_on_script_change, in Tokenizer() argument
69 internal_tokenizer_codepoint_ranges, split_on_script_change, in Tokenizer()
75 const bool split_on_script_change) in Tokenizer() argument
78 split_on_script_change,
|
| D | tokenizer.cc | 33 const bool split_on_script_change, in Tokenizer() argument
38 split_on_script_change_(split_on_script_change), in Tokenizer()
53 if (type_ == TokenizationType_MIXED && split_on_script_change) { in Tokenizer()
|
| D | tokenizer_test.cc | 36 const bool split_on_script_change, in TestingTokenizer() argument
40 internal_tokenizer_codepoint_ranges, split_on_script_change, in TestingTokenizer()
52 const bool split_on_script_change, in TestingTokenizerProxy() argument
80 type, &unilib_, configs_fb, internal_configs_fb, split_on_script_change, in TestingTokenizerProxy()
|