Searched refs:PreprocessToken (Results 1 – 3 of 3) sorted by relevance
38 void PreprocessToken(std::string& token) { in PreprocessToken() function87 PreprocessToken(token); in AddSkipgram()100 PreprocessToken(token); in AddSkipgram()120 PreprocessToken(token); in FindSkipgrams()
427 bool PreprocessToken = false; variable1048 void setPreprocessToken(bool Preprocess) { PreprocessToken = Preprocess; } in setPreprocessToken()
970 if ((LexLevel == 0 || PreprocessToken) && in Lex()