| /third_party/gn/src/gn/ |
| D | tokenizer_unittest.cc | 25 std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err); in CheckTokenizer()
45 std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err); in TEST()
51 results = Tokenizer::Tokenize(&whitespace_input, &err); in TEST()
127 std::vector<Token> results = Tokenizer::Tokenize(&input, &err); in TEST()
212 std::vector<Token> results = Tokenizer::Tokenize( in TEST()
224 Tokenizer::Tokenize(&input, &err, WhitespaceTransform::kInvalidToSpace); in TEST()
|
| D | parser_unittest.cc | 17 *result = Tokenizer::Tokenize(input, &err); in GetTokens()
62 std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); in DoParserErrorTest()
80 std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); in DoExpressionErrorTest()
|
| D | tokenizer.h | 29 static std::vector<Token> Tokenize(
|
| D | test_with_scope.cc | 75 std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, err); in ExecuteExpression()
333 tokens_ = Tokenizer::Tokenize(&input_file_, &parse_err_); in TestParseInput()
|
| D | input_file_manager.cc | 87 *tokens = Tokenizer::Tokenize(file, err); in DoLoadFile()
|
| /third_party/flutter/skia/third_party/externals/wuffs/lang/generate/ |
| D | generate.go | 106 tokens, _, err := t.Tokenize(tm, filename, src)
125 tokens, _, err := t.Tokenize(tm, filename, src)
|
| /third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/ |
| D | whitespace_tokenizer_op.h | 37 …Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32_t> *off…
|
| D | unicode_char_tokenizer_op.h | 36 …Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32_t> *off…
|
| D | unicode_script_tokenizer_op.h | 40 …Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32_t> *off…
|
| D | tokenizer_op.h | 37 …virtual Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32… in Tokenize() function
|
| D | tokenizer_op.cc | 34 RETURN_IF_NOT_OK(Tokenize(str, &splits, &offsets_start, &offsets_limit)); in Compute()
|
| D | unicode_char_tokenizer_op.cc | 29 Status UnicodeCharTokenizerOp::Tokenize(std::string_view str, std::vector<std::string> *splits, in Tokenize() function in mindspore::dataset::UnicodeCharTokenizerOp
|
| D | jieba_tokenizer_op.h | 50 …Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32_t> *off…
|
| D | regex_tokenizer_op.h | 45 …Status Tokenize(std::string_view str, std::vector<std::string> *splits, std::vector<uint32_t> *off…
|
| D | whitespace_tokenizer_op.cc | 30 Status WhitespaceTokenizerOp::Tokenize(std::string_view str, std::vector<std::string> *splits, in Tokenize() function in mindspore::dataset::WhitespaceTokenizerOp
|
| D | unicode_script_tokenizer_op.cc | 36 Status UnicodeScriptTokenizerOp::Tokenize(std::string_view str, std::vector<std::string> *splits, in Tokenize() function in mindspore::dataset::UnicodeScriptTokenizerOp
|
| D | jieba_tokenizer_op.cc | 32 Status JiebaTokenizerOp::Tokenize(std::string_view sentence_v, std::vector<std::string> *words, in Tokenize() function in mindspore::dataset::JiebaTokenizerOp
|
| D | regex_tokenizer_op.cc | 110 Status RegexTokenizerOp::Tokenize(std::string_view str, std::vector<std::string> *splits, in Tokenize() function in mindspore::dataset::RegexTokenizerOp
|
| /third_party/flutter/skia/third_party/externals/wuffs/lang/ast/ |
| D | string_test.go | 76 tokens, _, err := t.Tokenize(tm, filename, []byte(tc))
|
| /third_party/flutter/skia/third_party/externals/wuffs/lang/check/ |
| D | check_test.go | 99 tokens, comments, err := t.Tokenize(tm, filename, []byte(src))
215 tokens, _, err := t.Tokenize(tm, filename, []byte(src))
|
| D | resolve.go | 128 tokens, _, err := t.Tokenize(c.tm, filename, buf)
|
| /third_party/flutter/skia/third_party/externals/wuffs/cmd/wuffsfmt/ |
| D | main.go | 123 tokens, comments, err := t.Tokenize(tm, filename, src)
|
| /third_party/mindspore/mindspore/lite/src/common/ |
| D | utils.cc | 102 std::vector<std::string> Tokenize(const std::string &src, const std::string &delimiters, in Tokenize() function
|
| D | utils.h | 168 std::vector<std::string> Tokenize(const std::string &src, const std::string &delimiters,
|
| /third_party/flutter/skia/third_party/externals/wuffs/lang/token/ |
| D | token.go | 116 func Tokenize(m *Map, filename string, src []byte) (tokens []Token, comments []string, retErr error… func
|