Searched refs:total_tokens (Results 1 – 3 of 3) sorted by relevance
137 int64_t total_tokens = NumElements(input_values); in Eval() local140 tensor_input_row_splits.reserve(total_tokens / tokens_per_element + 1); in Eval()141 tensor_output_row_splits.resize(total_tokens / tokens_per_element + 1); in Eval()142 for (int64_t i = 0; i <= total_tokens; i += tokens_per_element) { in Eval()
139 int total_tokens = 0; in Eval() local144 total_tokens += num_message_tokens; in Eval()145 encoded_offsets.push_back(total_tokens); in Eval()
544 int total_tokens = 0; in EmbedAndFlattenTokens() local549 total_tokens += num_message_tokens; in EmbedAndFlattenTokens()552 if (total_tokens >= max_num_total_tokens) { in EmbedAndFlattenTokens()553 message_token_offset = total_tokens - max_num_total_tokens; in EmbedAndFlattenTokens()