Searched refs:remap_digits (Results 1 – 6 of 6) sorted by relevance
33 if (!options.remap_digits && !options.lowercase_tokens) { in RemapTokenAscii()39 if (options.remap_digits && isdigit(copy[i])) { in RemapTokenAscii()52 if (!options.remap_digits && !options.lowercase_tokens) { in RemapTokenUnicode()60 if (options.remap_digits && unilib.IsDigit(*it)) { in RemapTokenUnicode()
51 bool remap_digits = false; member
275 options.remap_digits = true; in TEST()300 options.remap_digits = true; in TEST()371 options.remap_digits = false; in TEST()
2440 bool remap_digits; member2470 remap_digits(false), in FeatureProcessorOptionsT()2554 bool remap_digits() const { in remap_digits() function2705 void add_remap_digits(bool remap_digits) { in add_remap_digits()2706 …dElement<uint8_t>(FeatureProcessorOptions::VT_REMAP_DIGITS, static_cast<uint8_t>(remap_digits), 0); in add_remap_digits()2790 bool remap_digits = false,2838 builder_.add_remap_digits(remap_digits);2858 bool remap_digits = false,2891 remap_digits,3590 { auto _e = remap_digits(); _o->remap_digits = _e; }; in UnPackTo()[all …]
51 extractor_options.remap_digits = options->remap_digits(); in BuildTokenFeatureExtractorOptions()
495 remap_digits:bool = 0;