Searched refs:out_tokens (Results 1 – 5 of 5) sorted by relevance
/third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/ |
D | wordpiece_tokenizer_op.cc | 58 … std::vector<std::string> *out_tokens, std::vector<uint32_t> *offsets_start, in FoundNoToken() argument 60 out_tokens->clear(); in FoundNoToken() 63 (void)out_tokens->emplace_back(input_token); in FoundNoToken() 66 (void)out_tokens->emplace_back(unknown_token_); in FoundNoToken() 73 std::vector<std::string> *out_tokens) const { in AddSubword() 80 (void)out_tokens->emplace_back(subword); in AddSubword() 85 … std::vector<std::string> *out_tokens, std::vector<uint32_t> *offsets_start, in GetTokens() argument 91 (void)out_tokens->emplace_back(unknown_token_); in GetTokens() 93 (void)out_tokens->emplace_back(input_token); in GetTokens() 107 RETURN_IF_NOT_OK(AddSubword(input_token, start, end, out_tokens)); in GetTokens() [all …]
|
D | regex_tokenizer_op.cc | 43 …tus RegexTokenizerOp::GetRegexTokens(const std::string &text, std::vector<std::string> *out_tokens, in GetRegexTokens() argument 47 out_tokens->clear(); in GetRegexTokens() 74 (void)out_tokens->emplace_back(std::move(token)); in GetRegexTokens() 89 (void)out_tokens->emplace_back(std::move(delim_utf8_str)); in GetRegexTokens() 103 (void)out_tokens->emplace_back(std::move(temp)); in GetRegexTokens()
|
D | wordpiece_tokenizer_op.h | 52 …(const std::string &input_token, const uint32_t &basic_start, std::vector<std::string> *out_tokens, 56 …(const std::string &input_token, const uint32_t &basic_start, std::vector<std::string> *out_tokens,
|
D | regex_tokenizer_op.h | 51 Status GetRegexTokens(const std::string &text, std::vector<std::string> *out_tokens,
|
/third_party/mesa3d/src/gallium/drivers/i915/ |
D | i915_fpc_optimize.c | 650 struct i915_token_list *out_tokens = MALLOC(sizeof(struct i915_token_list)); in i915_optimize() local 657 out_tokens->NumTokens = 0; in i915_optimize() 663 out_tokens->NumTokens++; in i915_optimize() 668 out_tokens->Tokens = in i915_optimize() 669 MALLOC(sizeof(union i915_full_token) * out_tokens->NumTokens); in i915_optimize() 676 out_tokens->NumTokens--; in i915_optimize() 680 copy_token(&out_tokens->Tokens[i], &parse.FullToken); in i915_optimize() 686 liveness_analysis(ctx, out_tokens); in i915_optimize() 689 while (i < out_tokens->NumTokens) { in i915_optimize() 690 i915_fpc_optimize_useless_mov_after_inst(ctx, out_tokens, i); in i915_optimize() [all …]
|