Searched defs:token_span (Results 1 – 4 of 4) sorted by relevance
215 TokenSpan token_span; in LabelToSpan() local370 const std::vector<Token>& selectable_tokens, TokenSpan token_span) { in TokenSpanToCodepointSpan()617 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio() argument690 TokenSpan token_span{l, r}; in MakeLabelMaps() local789 const std::vector<Token>& tokens, TokenSpan token_span, in ExtractFeatures()
166 TokenSpan token_span; member
94 inline int TokenSpanSize(const TokenSpan& token_span) { in TokenSpanSize()113 inline TokenSpan ExpandTokenSpan(const TokenSpan& token_span, in ExpandTokenSpan()
320 TokenSpan token_span; in TEST() local395 TokenSpan token_span; in TEST() local