• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_
17 #define THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_
18 
19 #include "tensorflow_lite_support/cc/port/statusor.h"
20 #include "tensorflow_lite_support/cc/text/tokenizers/tokenizer.h"
21 #include "tensorflow_lite_support/metadata/cc/metadata_extractor.h"
22 #include "tensorflow_lite_support/metadata/metadata_schema_generated.h"
23 
24 namespace tflite {
25 namespace support {
26 namespace text {
27 namespace tokenizer {
28 
29 
30 // Create a Tokenizer from model metadata by extracting
31 tflite::support::StatusOr<std::unique_ptr<Tokenizer>>
32 CreateTokenizerFromProcessUnit(
33     const tflite::ProcessUnit* tokenizer_process_unit,
34     const tflite::metadata::ModelMetadataExtractor* metadata_extractor);
35 
36 }  // namespace tokenizer
37 }  // namespace text
38 }  // namespace support
39 }  // namespace tflite
40 
41 #endif  // THIRD_PARTY_TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_TOKENIZER_UTILS_H_
42