Searched refs:tokenize_on_script_change (Results 1 – 7 of 7) sorted by relevance
46 const bool tokenize_on_script_change = in BuildTokenizer() local48 options->tokenize_on_script_change(); in BuildTokenizer()50 internal_codepoint_config, tokenize_on_script_change, in BuildTokenizer()
35 model->tokenizer_options->tokenize_on_script_change = true; in SetTestTokenizerOptions()
60 const bool tokenize_on_script_change = in CreateTokenizer() local62 options->tokenize_on_script_change(); in CreateTokenizer()65 tokenize_on_script_change, options->icu_preserve_whitespace_tokens())); in CreateTokenizer()
154 tokenize_on_script_change:bool = false;
373 tokenize_on_script_change:bool = false;790 tokenize_on_script_change:bool = false;
1810 grammar_model->tokenizer_options->tokenize_on_script_change = true; in TEST_F()