1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 // Util methods to read and write String tensors. 17 // String tensors are considered to be char tensor with protocol. 18 // [0, 3] 4 bytes: N, num of strings in the tensor in little endian. 19 // [(i+1)*4, (i+1)*4+3] 4 bytes: offset of i-th string in little endian. 20 // [(N+2)*4, (N+2)*4+3] 4 bytes: length of the whole char buffer. 21 // [offset(i), offset(i+1) - 1] : content of i-th string. 22 // Example of a string tensor: 23 // [ 24 // 2, 0, 0, 0, # 2 strings. 25 // 16, 0, 0, 0, # 0-th string starts from index 12. 26 // 18, 0, 0, 0, # 1-st string starts from index 18. 27 // 18, 0, 0, 0, # total length of array. 28 // 'A', 'B', # 0-th string [16..17]: "AB" 29 // ] # 1-th string, empty 30 // 31 // A typical usage: 32 // In op.Eval(context, node): 33 // DynamicBuffer buf; 34 // # Add string "AB" to tensor, string is stored in dynamic buffer. 35 // buf.AddString("AB", 2); 36 // # Write content of DynamicBuffer to tensor in format of string tensor 37 // # described above. 38 // buf.WriteToTensor(tensor) 39 40 #ifndef TENSORFLOW_CONTRIB_LITE_STRING_UTIL_H_ 41 #define TENSORFLOW_CONTRIB_LITE_STRING_UTIL_H_ 42 43 #include <vector> 44 45 #include "tensorflow/contrib/lite/context.h" 46 #include "tensorflow/contrib/lite/string_tflite.h" 47 48 namespace tflite { 49 50 // Convenient structure to store string pointer and length. 51 typedef struct { 52 char* str; 53 int len; 54 } StringRef; 55 56 // DynamicBuffer holds temporary buffer that will be used to create a dynamic 57 // tensor. A typical usage is to initialize a DynamicBuffer object, fill in 58 // content and call CreateStringTensor in op.Eval(). 59 class DynamicBuffer { 60 public: DynamicBuffer()61 DynamicBuffer() : offset_({0}) {} 62 63 // Add string to dynamic buffer by resizing the buffer and copying the data. 64 void AddString(const StringRef& string); 65 66 // Add string to dynamic buffer by resizing the buffer and copying the data. 67 void AddString(const char* str, size_t len); 68 69 // Join a list of string with separator, and add as a single string to the 70 // buffer. 71 void AddJoinedString(const std::vector<StringRef>& strings, char separator); 72 73 // Fill content into a string tensor. 74 void WriteToTensor(TfLiteTensor* tensor); 75 76 private: 77 // Data buffer to store contents of strings, not including headers. 78 std::vector<char> data_; 79 // Offset of the starting index of each string in data buffer. 80 std::vector<int32_t> offset_; 81 }; 82 83 // Return num of strings in a String tensor. 84 int GetStringCount(const TfLiteTensor* tensor); 85 86 // Get String pointer and length of index-th string in tensor. 87 // NOTE: This will not create a copy of string data. 88 StringRef GetString(const TfLiteTensor* tensor, int string_index); 89 } // namespace tflite 90 91 #endif // TENSORFLOW_CONTRIB_LITE_STRING_UTIL_H_ 92