| /external/libtextclassifier/annotator/duration/ |
| D | duration_test.cc | 108 std::vector<Token> Tokenize(const UnicodeText& text) { in Tokenize() function in libtextclassifier3::__anond123b13d0111::DurationAnnotatorTest
109 return feature_processor_.Tokenize(text); in Tokenize()
141 std::vector<Token> tokens = Tokenize(text); in TEST_F()
160 std::vector<Token> tokens = Tokenize(text); in TEST_F()
179 std::vector<Token> tokens = Tokenize(text); in TEST_F()
197 std::vector<Token> tokens = Tokenize(text); in TEST_F()
216 std::vector<Token> tokens = Tokenize(text); in TEST_F()
235 std::vector<Token> tokens = Tokenize(text); in TEST_F()
255 std::vector<Token> tokens = Tokenize(text); in TEST_F()
274 std::vector<Token> tokens = Tokenize(text); in TEST_F()
[all …]
|
| /external/libtextclassifier/utils/ |
| D | tokenizer_test.cc | 92 std::vector<Token> Tokenize(const std::string& utf8_text) const { in Tokenize() function in libtextclassifier3::__anonfe576b980111::TestingTokenizerProxy
93 return tokenizer_->Tokenize(utf8_text); in Tokenize()
174 std::vector<Token> tokens = tokenizer.Tokenize("Hello world!"); in TEST()
208 EXPECT_THAT(tokenizer.Tokenize("앨라배마 주 전화(123) 456-789웹사이트"), in TEST()
345 tokens = tokenizer.Tokenize( in TEST()
349 tokens = tokenizer.Tokenize("問少目 hello 木輸ยามきゃ"); in TEST()
372 std::vector<Token> tokens = tokenizer.Tokenize("พระบาทสมเด็จพระปรมิ"); in TEST()
387 std::vector<Token> tokens = tokenizer.Tokenize("พระบาท สมเด็จ พระ ปร มิ"); in TEST()
440 std::vector<Token> tokens = tokenizer.Tokenize( in TEST()
468 EXPECT_EQ(tokenizer.Tokenize("앨라배마123웹사이트"), in TEST()
[all …]
|
| D | tokenizer.h | 66 std::vector<Token> Tokenize(const std::string& text) const;
69 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
|
| D | tokenizer.cc | 90 std::vector<Token> Tokenizer::Tokenize(const std::string& text) const { in Tokenize() function in libtextclassifier3::Tokenizer
92 return Tokenize(text_unicode); in Tokenize()
95 std::vector<Token> Tokenizer::Tokenize(const UnicodeText& text_unicode) const { in Tokenize() function in libtextclassifier3::Tokenizer
|
| /external/perfetto/src/trace_processor/ |
| D | proto_trace_parser_unittest.cc | 165 void Tokenize() { in Tokenize() function in perfetto::trace_processor::__anon83c82cc70111::ProtoTraceParserTest
211 Tokenize(); in TEST_F()
247 Tokenize(); in TEST_F()
296 Tokenize(); in TEST_F()
359 Tokenize(); in TEST_F()
404 Tokenize(); in TEST_F()
426 Tokenize(); in TEST_F()
446 Tokenize(); in TEST_F()
461 Tokenize(); in TEST_F()
476 Tokenize(); in TEST_F()
[all …]
|
| /external/google-breakpad/src/processor/ |
| D | basic_source_line_resolver.cc | 471 if (!Tokenize(file_line, kWhitespace, 2, &tokens)) { in ParseFile()
499 if (!Tokenize(function_line, kWhitespace, 4, &tokens)) { in ParseFunction()
531 if (!Tokenize(line_line, kWhitespace, 4, &tokens)) { in ParseLine()
580 if (!Tokenize(public_line, kWhitespace, 3, &tokens)) { in ParsePublicSymbol()
|
| D | tokenize.h | 53 bool Tokenize(char *line,
|
| D | tokenize.cc | 45 bool Tokenize(char *line, in Tokenize() function
|
| D | windows_frame_info.h | 126 if (!Tokenize(&buffer[0], " \r\n", 11, &tokens)) in ParseFromString()
|
| /external/libtextclassifier/annotator/ |
| D | feature-processor_test.cc | 311 std::vector<Token> tokens = feature_processor.Tokenize("one, two, three"); in TEST_F()
352 tokens = feature_processor3.Tokenize("zero, one, two, three, four"); in TEST_F()
385 std::vector<Token> tokens = feature_processor.Tokenize("one, two, three"); in TEST_F()
426 tokens = feature_processor3.Tokenize("zero, one, two, three, four"); in TEST_F()
557 {0, 3}, feature_processor.Tokenize("aaa bbb ccc")), in TEST_F()
560 {0, 3}, feature_processor.Tokenize("aaa bbb ěěě")), in TEST_F()
563 {0, 3}, feature_processor.Tokenize("ěěě řřř ěěě")), in TEST_F()
|
| D | feature-processor.h | 109 std::vector<Token> Tokenize(const std::string& text) const;
112 std::vector<Token> Tokenize(const UnicodeText& text_unicode) const;
|
| D | feature-processor.cc | 191 std::vector<Token> FeatureProcessor::Tokenize(const std::string& text) const { in Tokenize() function in libtextclassifier3::FeatureProcessor
192 return tokenizer_.Tokenize(text); in Tokenize()
195 std::vector<Token> FeatureProcessor::Tokenize( in Tokenize() function in libtextclassifier3::FeatureProcessor
197 return tokenizer_.Tokenize(text_unicode); in Tokenize()
|
| /external/libtextclassifier/lang_id/ |
| D | custom-tokenizer.h | 42 void Tokenize(StringPiece text, LightSentence *sentence) const;
|
| D | custom-tokenizer.cc | 102 void TokenizerForLangId::Tokenize(StringPiece text, in Tokenize() function in libtextclassifier3::mobile::lang_id::TokenizerForLangId
|
| /external/tensorflow/tensorflow/lite/testing/nnapi_tflite_zip_tests/ |
| D | tokenize.h | 40 void Tokenize(std::istream* input, TokenProcessor* processor);
|
| D | message.cc | 94 Tokenize(input, &stack); in Read()
|
| D | tokenize.cc | 26 void Tokenize(std::istream* input, TokenProcessor* processor) { in Tokenize() function
|
| /external/tensorflow/tensorflow/lite/testing/ |
| D | tokenize.h | 37 void Tokenize(std::istream* input, TokenProcessor* processor);
|
| D | message.cc | 91 Tokenize(input, &stack); in Read()
|
| D | tokenize.cc | 23 void Tokenize(std::istream* input, TokenProcessor* processor) { in Tokenize() function
|
| D | tokenize_test.cc | 39 Tokenize(&ss, &collector); in TokenizeString()
|
| /external/python/cpython2/Misc/NEWS.d/next/Library/ |
| D | 2018-06-24-01-57-14.bpo-33899.IaOcAr.rst | 1 Tokenize module now implicitly emits a NEWLINE when provided with input that
|
| /external/chromium-trace/catapult/common/py_utils/py_utils/refactor/ |
| D | offset_token.py | 55 def Tokenize(f): function
|
| /external/libtextclassifier/annotator/number/ |
| D | number.cc | 57 const std::vector<Token> tokens = feature_processor_->Tokenize(context); in FindAll()
|
| /external/libtextclassifier/actions/ |
| D | ngram-model.cc | 150 const std::vector<Token> raw_tokens = tokenizer_->Tokenize(text); in Eval()
|