Searched refs:token_id (Results 1 – 4 of 4) sorted by relevance
353 TokenId token_id = token_info.id << 1; in Parse() local355 token_id |= (it == sym_end) ? 1 : 0; // Last one has LSB set to 1. in Parse()356 wptr = protozero::proto_utils::WriteVarInt(token_id, wptr); in Parse()
228 int token_id = 0; in Preprocess() local229 if (tokenizer_->LookupId(token, &token_id)) { in Preprocess()230 input_tokens[input_token_index] = token_id; in Preprocess()
118 bool EmbedTokenId(const int32 token_id, std::vector<float>* embedding) const;
467 bool ActionsSuggestions::EmbedTokenId(const int32 token_id, in EmbedTokenId() argument470 {token_id}, in EmbedTokenId()