Searched refs:query_tokens (Results 1 – 2 of 2) sorted by relevance
201 std::vector<std::string> query_tokens = query_tokenize_results.subwords; in Preprocess() local202 if (query_tokens.size() > kMaxQueryLen) { in Preprocess()203 query_tokens.resize(kMaxQueryLen); in Preprocess()223 int max_context_len = kMaxSeqLen - query_tokens.size() - 3; in Preprocess()229 tokens.reserve(3 + query_tokens.size() + all_doc_tokens.size()); in Preprocess()238 for (const auto& query_token : query_tokens) { in Preprocess()
82 absl::Span<const std::string> query_tokens = in Preprocess() local89 tokens.reserve(2 + query_tokens.size()); in Preprocess()93 for (const auto& query_token : query_tokens) { in Preprocess()