Searched refs:token_span (Results 1 – 5 of 5) sorted by relevance
| /external/libtextclassifier/native/annotator/ |
| D | feature-processor.cc | 205 TokenSpan token_span; in LabelToSpan() local
206 if (!LabelToTokenSpan(label, &token_span)) { in LabelToSpan()
210 const int result_begin_token_index = token_span.first; in LabelToSpan()
214 const int result_end_token_index = token_span.second; in LabelToSpan()
249 TokenSpan* token_span) const { in LabelToTokenSpan()
251 *token_span = label_to_selection_[label]; in LabelToTokenSpan()
323 int FeatureProcessor::TokenSpanToLabel(const TokenSpan& token_span) const { in TokenSpanToLabel()
324 auto it = selection_to_label_.find(token_span); in TokenSpanToLabel()
360 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span) { in TokenSpanToCodepointSpan() argument
361 return {selectable_tokens[token_span.first].start, in TokenSpanToCodepointSpan()
[all …]
|
| D | feature-processor.h | 84 const std::vector<Token>& selectable_tokens, const TokenSpan& token_span);
130 bool LabelToTokenSpan(int label, TokenSpan* token_span) const;
165 const TokenSpan& token_span) const;
170 const TokenSpan& token_span,
247 int TokenSpanToLabel(const TokenSpan& token_span) const;
251 float SupportedCodepointsRatio(const TokenSpan& token_span,
|
| D | feature-processor_test.cc | 390 TokenSpan token_span; in TEST_F() local
391 feature_processor.LabelToTokenSpan(label, &token_span); in TEST_F()
392 EXPECT_EQ(0, token_span.first); in TEST_F()
393 EXPECT_EQ(0, token_span.second); in TEST_F()
427 feature_processor3.LabelToTokenSpan(label2, &token_span); in TEST_F()
428 EXPECT_EQ(1, token_span.first); in TEST_F()
429 EXPECT_EQ(0, token_span.second); in TEST_F()
464 TokenSpan token_span; in TEST_F() local
465 feature_processor.LabelToTokenSpan(label, &token_span); in TEST_F()
466 EXPECT_EQ(0, token_span.first); in TEST_F()
[all …]
|
| D | annotator.cc | 2938 for (int i = scored_chunk.token_span.first; in ModelChunk()
2939 i < scored_chunk.token_span.second; ++i) { in ModelChunk()
2950 for (int i = scored_chunk.token_span.first; in ModelChunk()
2951 i < scored_chunk.token_span.second; ++i) { in ModelChunk()
2955 chunks->push_back(scored_chunk.token_span); in ModelChunk()
|
| D | annotator.h | 245 TokenSpan token_span; member
|