Lines Matching refs:CodepointSpan
49 void SplitTokensOnSelectionBoundaries(CodepointSpan selection,
53 int CenterTokenFromClick(CodepointSpan span, const std::vector<Token>& tokens);
58 CodepointSpan span, const std::vector<Token>& selectable_tokens);
79 const std::vector<Token>& selectable_tokens, CodepointSpan codepoint_span,
83 CodepointSpan TokenSpanToCodepointSpan(
94 typedef std::map<CodepointSpan, std::vector<float>> EmbeddingCache;
152 CodepointSpan input_span,
158 CodepointSpan input_span,
170 CodepointSpan selection_span_for_feature,
180 std::vector<CodepointSpan>* selection_label_spans) const;
195 CodepointSpan StripBoundaryCodepoints(const std::string& context,
196 CodepointSpan span) const;
199 CodepointSpan StripBoundaryCodepoints(const UnicodeText& context_unicode,
200 CodepointSpan span) const;
230 CodepointSpan* span) const;
268 int FindCenterToken(CodepointSpan span,
283 void TokenizeSubstring(const UnicodeText& unicode_text, CodepointSpan span,
288 void StripTokensFromOtherLines(const std::string& context, CodepointSpan span,
293 CodepointSpan span,
300 CodepointSpan selection_span_for_feature,