Searched refs:tokenizer_process_unit (Results 1 – 5 of 5) sorted by relevance
59 const tflite::ProcessUnit* tokenizer_process_unit, in CreateTokenizerFromProcessUnit() argument61 if (metadata_extractor == nullptr || tokenizer_process_unit == nullptr) { in CreateTokenizerFromProcessUnit()67 switch (tokenizer_process_unit->options_type()) { in CreateTokenizerFromProcessUnit()70 tokenizer_process_unit->options_as<tflite::BertTokenizerOptions>(); in CreateTokenizerFromProcessUnit()79 tokenizer_process_unit->options_as<SentencePieceTokenizerOptions>(); in CreateTokenizerFromProcessUnit()89 tokenizer_process_unit->options_as<RegexTokenizerOptions>(); in CreateTokenizerFromProcessUnit()127 tokenizer_process_unit->options_type()), in CreateTokenizerFromProcessUnit()
33 const tflite::ProcessUnit* tokenizer_process_unit,
176 const ProcessUnit* tokenizer_process_unit = in InitializeFromMetadata() local178 if (tokenizer_process_unit == nullptr) { in InitializeFromMetadata()185 CreateTokenizerFromProcessUnit(tokenizer_process_unit, in InitializeFromMetadata()
85 const tflite::ProcessUnit* tokenizer_process_unit, in CreateRegexTokenizerFromProcessUnit() argument87 if (metadata_extractor == nullptr || tokenizer_process_unit == nullptr) { in CreateRegexTokenizerFromProcessUnit()94 if (tokenizer_process_unit->options_type() != in CreateRegexTokenizerFromProcessUnit()99 "Incorrect options_type:", tokenizer_process_unit->options_type(), in CreateRegexTokenizerFromProcessUnit()105 tokenizer_process_unit->options_as<RegexTokenizerOptions>(); in CreateRegexTokenizerFromProcessUnit()
354 const ProcessUnit* tokenizer_process_unit = in InitializeFromMetadata() local356 if (tokenizer_process_unit == nullptr) { in InitializeFromMetadata()363 CreateTokenizerFromProcessUnit(tokenizer_process_unit, in InitializeFromMetadata()