Home
last modified time | relevance | path

Searched refs:TokenSpan (Results 1 – 9 of 9) sorted by relevance

/external/libtextclassifier/annotator/
Dcached-features.h34 const TokenSpan& extraction_span,
47 TokenSpan selected_span, std::vector<float>* output_features) const;
59 void AppendFeaturesInternal(const TokenSpan& intended_span,
60 const TokenSpan& read_mask_span,
69 void AppendBagFeatures(const TokenSpan& bag_span,
74 TokenSpan extraction_span_;
Dfeature-processor.h67 void StripOrPadTokens(TokenSpan relative_click_span, int context_size,
76 TokenSpan CodepointSpanToTokenSpan(
82 const std::vector<Token>& selectable_tokens, TokenSpan token_span);
115 bool LabelToTokenSpan(int label, TokenSpan* token_span) const;
147 TokenSpan token_span) const;
151 bool ExtractFeatures(const std::vector<Token>& tokens, TokenSpan token_span,
225 float SupportedCodepointsRatio(const TokenSpan& token_span,
279 std::map<TokenSpan, int> selection_to_label_;
280 std::vector<TokenSpan> label_to_selection_;
Dcached-features.cc60 const TokenSpan& extraction_span, in Create()
98 TokenSpan selected_span, std::vector<float>* output_features) const { in AppendBoundsSensitiveFeaturesForSpan()
135 const TokenSpan& intended_span, const TokenSpan& read_mask_span, in AppendFeaturesInternal()
137 const TokenSpan copy_span = in AppendFeaturesInternal()
158 const TokenSpan& bag_span, std::vector<float>* output_features) const { in AppendBagFeatures()
Dtypes.h96 using TokenSpan = std::pair<TokenIndex, TokenIndex>; variable
99 inline int TokenSpanSize(const TokenSpan& token_span) { in TokenSpanSize()
104 inline TokenSpan SingleTokenSpan(int token_index) { in SingleTokenSpan()
110 inline TokenSpan IntersectTokenSpans(const TokenSpan& token_span1, in IntersectTokenSpans()
111 const TokenSpan& token_span2) { in IntersectTokenSpans()
118 inline TokenSpan ExpandTokenSpan(const TokenSpan& token_span, in ExpandTokenSpan()
Dfeature-processor_test.cc318 TokenSpan token_span; in TEST_F()
392 TokenSpan token_span; in TEST_F()
950 EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5})); in TEST_F()
951 EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23})); in TEST_F()
952 EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29})); in TEST_F()
953 EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23})); in TEST_F()
954 EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29})); in TEST_F()
955 EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29})); in TEST_F()
958 EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}, true)); in TEST_F()
959 EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}, true)); in TEST_F()
[all …]
Dannotator.h276 TokenSpan token_span;
363 TokenSpan ClassifyTextUpperBoundNeededTokens() const;
399 bool ModelChunk(int num_tokens, const TokenSpan& span_of_interest,
402 std::vector<TokenSpan>* chunks) const;
408 int num_tokens, const TokenSpan& span_of_interest,
417 int num_tokens, const TokenSpan& span_of_interest,
418 const TokenSpan& inference_span, const CachedFeatures& cached_features,
539 TokenSpan tokens_around_selection_to_copy);
Dfeature-processor.cc207 TokenSpan token_span; in LabelToSpan()
251 TokenSpan* token_span) const { in LabelToTokenSpan()
325 int FeatureProcessor::TokenSpanToLabel(const TokenSpan& span) const { in TokenSpanToLabel()
334 TokenSpan CodepointSpanToTokenSpan(const std::vector<Token>& selectable_tokens, in CodepointSpanToTokenSpan()
362 const std::vector<Token>& selectable_tokens, TokenSpan token_span) { in TokenSpanToCodepointSpan()
600 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio()
662 TokenSpan token_span{l, r}; in MakeLabelMaps()
710 void StripOrPadTokens(TokenSpan relative_click_span, int context_size, in StripOrPadTokens()
747 const std::vector<Token>& tokens, TokenSpan token_span) const { in HasEnoughSupportedCodepoints()
761 const std::vector<Token>& tokens, TokenSpan token_span, in ExtractFeatures()
Dannotator.cc1038 const TokenSpan symmetry_context_span = IntersectTokenSpans( in ModelSuggestSelection()
1045 TokenSpan extraction_span; in ModelSuggestSelection()
1089 std::vector<TokenSpan> chunks; in ModelSuggestSelection()
1097 for (const TokenSpan& chunk : chunks) { in ModelSuggestSelection()
1128 TokenSpan tokens_around_selection_to_copy) { in CopyCachedTokens()
1158 TokenSpan Annotator::ClassifyTextUpperBoundNeededTokens() const { in ClassifyTextUpperBoundNeededTokens()
1232 const TokenSpan selection_token_span = in ModelClassifyText()
1253 TokenSpan extraction_span; in ModelClassifyText()
1674 const TokenSpan full_line_span = {0, tokens->size()}; in ModelAnnotate()
1695 std::vector<TokenSpan> local_chunks; in ModelAnnotate()
[all …]
Dcached-features_test.cc59 const CachedFeatures& cached_features, TokenSpan selected_span) { in GetCachedBoundsSensitiveFeatures()