| /external/libtextclassifier/native/annotator/ |
| D | cached-features.h | 34 const TokenSpan& extraction_span,
47 TokenSpan selected_span, std::vector<float>* output_features) const;
59 void AppendFeaturesInternal(const TokenSpan& intended_span,
60 const TokenSpan& read_mask_span,
69 void AppendBagFeatures(const TokenSpan& bag_span,
74 TokenSpan extraction_span_;
|
| D | feature-processor.h | 68 void StripOrPadTokens(const TokenSpan& relative_click_span, int context_size,
77 TokenSpan CodepointSpanToTokenSpan(
84 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span);
130 bool LabelToTokenSpan(int label, TokenSpan* token_span) const;
165 const TokenSpan& token_span) const;
170 const TokenSpan& token_span,
186 std::vector<TokenSpan>* selection_label_relative_token_spans) const;
247 int TokenSpanToLabel(const TokenSpan& token_span) const;
251 float SupportedCodepointsRatio(const TokenSpan& token_span,
308 std::map<TokenSpan, int> selection_to_label_;
[all …]
|
| D | cached-features.cc | 60 const TokenSpan& extraction_span, in Create()
91 /*intended_span=*/TokenSpan(click_pos).Expand(options_->context_size(), in AppendClickContextFeaturesForClick()
97 TokenSpan selected_span, std::vector<float>* output_features) const { in AppendBoundsSensitiveFeaturesForSpan()
133 const TokenSpan& intended_span, const TokenSpan& read_mask_span, in AppendFeaturesInternal()
135 const TokenSpan copy_span = in AppendFeaturesInternal()
156 const TokenSpan& bag_span, std::vector<float>* output_features) const { in AppendBagFeatures()
|
| D | types.h | 144 struct TokenSpan { struct
145 static const TokenSpan kInvalid; argument
147 TokenSpan() : first(kInvalidIndex), second(kInvalidIndex) {} in TokenSpan() argument
149 TokenSpan(TokenIndex start, TokenIndex end) : first(start), second(end) {} in TokenSpan() function
152 explicit TokenSpan(int token_index) in TokenSpan() argument
155 TokenSpan& operator=(const TokenSpan& other) = default; argument
157 bool operator==(const TokenSpan& other) const {
161 bool operator!=(const TokenSpan& other) const { return !(*this == other); }
163 bool operator<(const TokenSpan& other) const {
179 TokenSpan Expand(int num_tokens_left, int num_tokens_right) const { in Expand() argument
[all …]
|
| D | feature-processor_test.cc | 390 TokenSpan token_span; in TEST_F()
464 TokenSpan token_span; in TEST_F()
1025 EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5})); in TEST_F()
1026 EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23})); in TEST_F()
1027 EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29})); in TEST_F()
1028 EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23})); in TEST_F()
1029 EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29})); in TEST_F()
1030 EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29})); in TEST_F()
1033 EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}, true)); in TEST_F()
1034 EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}, true)); in TEST_F()
[all …]
|
| D | feature-processor.cc | 205 TokenSpan token_span; in LabelToSpan()
249 TokenSpan* token_span) const { in LabelToTokenSpan()
323 int FeatureProcessor::TokenSpanToLabel(const TokenSpan& token_span) const { in TokenSpanToLabel()
332 TokenSpan CodepointSpanToTokenSpan(const std::vector<Token>& selectable_tokens, in CodepointSpanToTokenSpan()
360 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span) { in TokenSpanToCodepointSpan()
401 const TokenSpan token_span = in CenterTokenFromClick()
428 const TokenSpan token_span = in CenterTokenFromMiddleOfSelection()
488 std::vector<TokenSpan>* selection_label_relative_token_spans) const { in SelectionLabelRelativeTokenSpans()
621 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio()
687 TokenSpan token_span{l, r}; in MakeLabelMaps()
[all …]
|
| D | annotator.h | 245 TokenSpan token_span;
331 TokenSpan ClassifyTextUpperBoundNeededTokens() const;
368 bool ModelChunk(int num_tokens, const TokenSpan& span_of_interest,
371 std::vector<TokenSpan>* chunks) const;
377 int num_tokens, const TokenSpan& span_of_interest,
386 int num_tokens, const TokenSpan& span_of_interest,
387 const TokenSpan& inference_span, const CachedFeatures& cached_features,
581 TokenSpan tokens_around_selection_to_copy);
|
| D | annotator.cc | 1280 const TokenSpan symmetry_context_span = in ModelSuggestSelection()
1281 IntersectTokenSpans(TokenSpan(click_pos).Expand( in ModelSuggestSelection()
1287 TokenSpan extraction_span; in ModelSuggestSelection()
1330 std::vector<TokenSpan> chunks; in ModelSuggestSelection()
1338 for (const TokenSpan& chunk : chunks) { in ModelSuggestSelection()
1358 TokenSpan tokens_around_selection_to_copy) { in CopyCachedTokens()
1388 TokenSpan Annotator::ClassifyTextUpperBoundNeededTokens() const { in ClassifyTextUpperBoundNeededTokens()
1477 const TokenSpan selection_token_span = in ModelClassifyText()
1498 TokenSpan extraction_span; in ModelClassifyText()
1514 extraction_span = TokenSpan(click_pos).Expand( in ModelClassifyText()
[all …]
|
| D | types.cc | 28 const TokenSpan TokenSpan::kInvalid = TokenSpan(kInvalidIndex, kInvalidIndex);
36 const TokenSpan& span) { in operator <<()
|
| D | types-test-util.h | 41 TC3_DECLARE_PRINT_OPERATOR(TokenSpan)
|
| D | cached-features_test.cc | 59 const CachedFeatures& cached_features, TokenSpan selected_span) { in GetCachedBoundsSensitiveFeatures()
|
| /external/libtextclassifier/native/utils/grammar/ |
| D | text-context.h | 52 TokenSpan context_span;
|
| /external/libtextclassifier/native/annotator/grammar/ |
| D | grammar-annotator.cc | 279 if (const TokenSpan context_span = CodepointSpanToTokenSpan( in ClassifyText()
|