Searched defs:token_span (Results 1 – 4 of 4) sorted by relevance
207 TokenSpan token_span; in LabelToSpan() local362 const std::vector<Token>& selectable_tokens, TokenSpan token_span) { in TokenSpanToCodepointSpan()600 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio() argument662 TokenSpan token_span{l, r}; in MakeLabelMaps() local761 const std::vector<Token>& tokens, TokenSpan token_span, in ExtractFeatures()
318 TokenSpan token_span; in TEST_F() local392 TokenSpan token_span; in TEST_F() local
276 TokenSpan token_span; member
99 inline int TokenSpanSize(const TokenSpan& token_span) { in TokenSpanSize()118 inline TokenSpan ExpandTokenSpan(const TokenSpan& token_span, in ExpandTokenSpan()