Home
last modified time | relevance | path

Searched refs:out_tokens (Results 1 – 5 of 5) sorted by relevance

/third_party/mindspore/mindspore/ccsrc/minddata/dataset/text/kernels/
Dwordpiece_tokenizer_op.cc58 … std::vector<std::string> *out_tokens, std::vector<uint32_t> *offsets_start, in FoundNoToken() argument
60 out_tokens->clear(); in FoundNoToken()
63 (void)out_tokens->emplace_back(input_token); in FoundNoToken()
66 (void)out_tokens->emplace_back(unknown_token_); in FoundNoToken()
73 std::vector<std::string> *out_tokens) const { in AddSubword()
80 (void)out_tokens->emplace_back(subword); in AddSubword()
85 … std::vector<std::string> *out_tokens, std::vector<uint32_t> *offsets_start, in GetTokens() argument
91 (void)out_tokens->emplace_back(unknown_token_); in GetTokens()
93 (void)out_tokens->emplace_back(input_token); in GetTokens()
107 RETURN_IF_NOT_OK(AddSubword(input_token, start, end, out_tokens)); in GetTokens()
[all …]
Dregex_tokenizer_op.cc43 …tus RegexTokenizerOp::GetRegexTokens(const std::string &text, std::vector<std::string> *out_tokens, in GetRegexTokens() argument
47 out_tokens->clear(); in GetRegexTokens()
74 (void)out_tokens->emplace_back(std::move(token)); in GetRegexTokens()
89 (void)out_tokens->emplace_back(std::move(delim_utf8_str)); in GetRegexTokens()
103 (void)out_tokens->emplace_back(std::move(temp)); in GetRegexTokens()
Dwordpiece_tokenizer_op.h52 …(const std::string &input_token, const uint32_t &basic_start, std::vector<std::string> *out_tokens,
56 …(const std::string &input_token, const uint32_t &basic_start, std::vector<std::string> *out_tokens,
Dregex_tokenizer_op.h51 Status GetRegexTokens(const std::string &text, std::vector<std::string> *out_tokens,
/third_party/mesa3d/src/gallium/drivers/i915/
Di915_fpc_optimize.c650 struct i915_token_list *out_tokens = MALLOC(sizeof(struct i915_token_list)); in i915_optimize() local
657 out_tokens->NumTokens = 0; in i915_optimize()
663 out_tokens->NumTokens++; in i915_optimize()
668 out_tokens->Tokens = in i915_optimize()
669 MALLOC(sizeof(union i915_full_token) * out_tokens->NumTokens); in i915_optimize()
676 out_tokens->NumTokens--; in i915_optimize()
680 copy_token(&out_tokens->Tokens[i], &parse.FullToken); in i915_optimize()
686 liveness_analysis(ctx, out_tokens); in i915_optimize()
689 while (i < out_tokens->NumTokens) { in i915_optimize()
690 i915_fpc_optimize_useless_mov_after_inst(ctx, out_tokens, i); in i915_optimize()
[all …]