Searched refs:tokenize_on_script_change (Results 1 – 4 of 4) sorted by relevance
60 const bool tokenize_on_script_change = in CreateTokenizer() local62 options->tokenize_on_script_change(); in CreateTokenizer()65 tokenize_on_script_change, options->icu_preserve_whitespace_tokens())); in CreateTokenizer()
107 tokenize_on_script_change:bool = false;
46 const bool tokenize_on_script_change = in BuildTokenizer() local48 options->tokenize_on_script_change(); in BuildTokenizer()50 internal_codepoint_config, tokenize_on_script_change, in BuildTokenizer()
623 tokenize_on_script_change:bool = false;