1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_ 17 #define THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_ 18 19 #include "tensorflow_lite_support/cc/port/statusor.h" 20 #include "tensorflow_lite_support/cc/text/tokenizers/tokenizer.h" 21 #include "tensorflow_lite_support/metadata/cc/metadata_extractor.h" 22 #include "tensorflow_lite_support/metadata/metadata_schema_generated.h" 23 24 namespace tflite { 25 namespace support { 26 namespace text { 27 namespace tokenizer { 28 29 30 // Create a Tokenizer from model metadata by extracting 31 tflite::support::StatusOr<std::unique_ptr<Tokenizer>> 32 CreateTokenizerFromProcessUnit( 33 const tflite::ProcessUnit* tokenizer_process_unit, 34 const tflite::metadata::ModelMetadataExtractor* metadata_extractor); 35 36 } // namespace tokenizer 37 } // namespace text 38 } // namespace support 39 } // namespace tflite 40 41 #endif // THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_ 42