Searched refs:selectable_tokens (Results 1 – 2 of 2) sorted by relevance
332 TokenSpan CodepointSpanToTokenSpan(const std::vector<Token>& selectable_tokens, in CodepointSpanToTokenSpan() argument340 for (int i = 0; i < selectable_tokens.size(); ++i) { in CodepointSpanToTokenSpan()343 is_token_in_span = codepoint_start < selectable_tokens[i].end && in CodepointSpanToTokenSpan()344 codepoint_end > selectable_tokens[i].start; in CodepointSpanToTokenSpan()346 is_token_in_span = codepoint_start <= selectable_tokens[i].start && in CodepointSpanToTokenSpan()347 codepoint_end >= selectable_tokens[i].end; in CodepointSpanToTokenSpan()349 if (is_token_in_span && !selectable_tokens[i].is_padding) { in CodepointSpanToTokenSpan()360 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span) { in TokenSpanToCodepointSpan() argument361 return {selectable_tokens[token_span.first].start, in TokenSpanToCodepointSpan()362 selectable_tokens[token_span.second - 1].end}; in TokenSpanToCodepointSpan()[all …]
62 const CodepointSpan& span, const std::vector<Token>& selectable_tokens);78 const std::vector<Token>& selectable_tokens,84 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span);