Home
last modified time | relevance | path

Searched refs:token_starts (Results 1 – 5 of 5) sorted by relevance

/external/libtextclassifier/native/annotator/pod_ner/
Dpod-ner-impl.cc271 const VectorSpan<int32_t> &token_starts, in ExecuteModel() argument
279 if (wordpiece_indices.size() <= 0 || token_starts.size() <= 0 || in ExecuteModel()
283 << " #token_starts=" << token_starts.size() in ExecuteModel()
315 {1, token_starts.size()}); in ExecuteModel()
334 for (int i = 0; i < token_starts.size(); ++i) { in ExecuteModel()
337 tensor->data.i32[i] = token_starts[i] + 1 - token_starts[0]; in ExecuteModel()
348 std::vector<int32_t> *token_starts, in PrepareText() argument
381 token_starts->push_back(wordpiece_indices->size()); in PrepareText()
402 std::vector<int32_t> token_starts; in AnnotateAroundSpanOfInterest() local
404 if (!PrepareText(context, &wordpiece_indices, &token_starts, &tokens)) { in AnnotateAroundSpanOfInterest()
[all …]
Dpod-ner-impl.h62 const VectorSpan<int32_t> &token_starts,
67 std::vector<int32_t> *token_starts,
Dutils.cc154 const std::vector<int32_t> &token_starts, in WindowGenerator() argument
160 token_starts_(&token_starts), in WindowGenerator()
165 span_of_interest, tokens, token_starts, wordpiece_indices.size(), in WindowGenerator()
Dutils_test.cc851 std::vector<int32_t> token_starts{0, 2, 3, 5, 6, 7}; in TEST() local
852 WindowGenerator window_generator(wordpiece_indices, token_starts, tokens, in TEST()
869 ASSERT_EQ(cur_token_starts[i], token_starts[i]); in TEST()
883 ASSERT_EQ(cur_token_starts[i], token_starts[i + 1]); in TEST()
897 ASSERT_EQ(cur_token_starts[i], token_starts[i + 3]); in TEST()
Dutils.h121 const std::vector<int32_t> &token_starts,