Searched refs:token_span (Results 1 – 6 of 6) sorted by relevance
| /external/libtextclassifier/annotator/ |
| D | feature-processor.cc | 207 TokenSpan token_span; in LabelToSpan() local
208 if (!LabelToTokenSpan(label, &token_span)) { in LabelToSpan()
212 const int result_begin_token_index = token_span.first; in LabelToSpan()
216 const int result_end_token_index = token_span.second; in LabelToSpan()
251 TokenSpan* token_span) const { in LabelToTokenSpan()
253 *token_span = label_to_selection_[label]; in LabelToTokenSpan()
362 const std::vector<Token>& selectable_tokens, TokenSpan token_span) { in TokenSpanToCodepointSpan() argument
363 return {selectable_tokens[token_span.first].start, in TokenSpanToCodepointSpan()
364 selectable_tokens[token_span.second - 1].end}; in TokenSpanToCodepointSpan()
600 const TokenSpan& token_span, const std::vector<Token>& tokens) const { in SupportedCodepointsRatio() argument
[all …]
|
| D | feature-processor.h | 82 const std::vector<Token>& selectable_tokens, TokenSpan token_span);
115 bool LabelToTokenSpan(int label, TokenSpan* token_span) const;
147 TokenSpan token_span) const;
151 bool ExtractFeatures(const std::vector<Token>& tokens, TokenSpan token_span,
225 float SupportedCodepointsRatio(const TokenSpan& token_span,
|
| D | types.h | 99 inline int TokenSpanSize(const TokenSpan& token_span) { in TokenSpanSize() argument
100 return token_span.second - token_span.first; in TokenSpanSize()
118 inline TokenSpan ExpandTokenSpan(const TokenSpan& token_span, in ExpandTokenSpan() argument
120 return {token_span.first - num_tokens_left, in ExpandTokenSpan()
121 token_span.second + num_tokens_right}; in ExpandTokenSpan()
|
| D | feature-processor_test.cc | 318 TokenSpan token_span; in TEST_F() local
319 feature_processor.LabelToTokenSpan(label, &token_span); in TEST_F()
320 EXPECT_EQ(0, token_span.first); in TEST_F()
321 EXPECT_EQ(0, token_span.second); in TEST_F()
355 feature_processor3.LabelToTokenSpan(label2, &token_span); in TEST_F()
356 EXPECT_EQ(1, token_span.first); in TEST_F()
357 EXPECT_EQ(0, token_span.second); in TEST_F()
392 TokenSpan token_span; in TEST_F() local
393 feature_processor.LabelToTokenSpan(label, &token_span); in TEST_F()
394 EXPECT_EQ(0, token_span.first); in TEST_F()
[all …]
|
| D | annotator.cc | 2119 for (int i = scored_chunk.token_span.first; in ModelChunk()
2120 i < scored_chunk.token_span.second; ++i) { in ModelChunk()
2131 for (int i = scored_chunk.token_span.first; in ModelChunk()
2132 i < scored_chunk.token_span.second; ++i) { in ModelChunk()
2136 chunks->push_back(scored_chunk.token_span); in ModelChunk()
|
| D | annotator.h | 276 TokenSpan token_span; member
|