Searched refs:feature_processor_ (Results 1 – 8 of 8) sorted by relevance
34 feature_processor_(feature_processor), in VocabAnnotator()64 std::vector<Token> tokens = feature_processor_.Tokenize(context); in Annotate()103 feature_processor_.StripBoundaryCodepoints(context, in ClassifyTextInternal()
65 const FeatureProcessor &feature_processor_; variable
136 const std::vector<Token> tokens = feature_processor_->Tokenize(selection); in ClassifyText()249 result->span = feature_processor_->StripBoundaryCodepoints( in FindDurationStartingAt()304 const std::string& token_value = feature_processor_->StripBoundaryCodepoints( in ParseQuantityToken()327 const std::string& token_value = feature_processor_->StripBoundaryCodepoints( in ParseDurationUnitToken()369 const std::string& token_value = feature_processor_->StripBoundaryCodepoints( in ParseFillerToken()
108 feature_processor_(BuildFeatureProcessor(&unilib_)), in DurationAnnotatorTest()110 feature_processor_.get(), &unilib_) {} in DurationAnnotatorTest()113 return feature_processor_->Tokenize(text); in Tokenize()117 std::unique_ptr<FeatureProcessor> feature_processor_; member in libtextclassifier3::__anoncdfecbf10111::DurationAnnotatorTest525 feature_processor_(BuildFeatureProcessor(&unilib_)), in JapaneseDurationAnnotatorTest()527 feature_processor_.get(), &unilib_) {} in JapaneseDurationAnnotatorTest()530 return feature_processor_->Tokenize(text); in Tokenize()534 std::unique_ptr<FeatureProcessor> feature_processor_; member in libtextclassifier3::__anoncdfecbf10111::JapaneseDurationAnnotatorTest
71 feature_processor_(feature_processor), in DurationAnnotator()132 const FeatureProcessor* feature_processor_; variable
371 feature_processor_.reset(new ActionsFeatureProcessor(options, unilib_)); in ValidateAndInitialize()388 token_embedding_size_ = feature_processor_->GetTokenEmbeddingSize(); in ValidateAndInitialize()395 feature_processor_ == nullptr ? nullptr in ValidateAndInitialize()396 : feature_processor_->tokenizer()); in ValidateAndInitialize()469 return feature_processor_->AppendFeatures( in EmbedTokenId()479 tokens.push_back(feature_processor_->tokenizer()->Tokenize(message)); in Tokenize()516 if (!feature_processor_->AppendTokenFeatures( in EmbedTokensPerMessage()572 if (!feature_processor_->AppendTokenFeatures( in EmbedAndFlattenTokens()657 if (feature_processor_ == nullptr) { in SetupModelInput()
140 std::unique_ptr<const ActionsFeatureProcessor> feature_processor_; variable
1403 feature_processor_.reset(new ActionsFeatureProcessor(options, unilib_.get())); in TestingMessageEmbedder()1409 token_embedding_size_ = feature_processor_->GetTokenEmbeddingSize(); in TestingMessageEmbedder()