1 /* Copyright 2021 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_MINI_BENCHMARK_LIBJPEG_DECODER_H_ 16 #define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_MINI_BENCHMARK_LIBJPEG_DECODER_H_ 17 18 #include <memory.h> 19 20 #include <csetjmp> 21 #include <cstddef> 22 #include <memory> 23 #include <string> 24 #include <type_traits> 25 #include <utility> 26 27 #include "tensorflow/lite/c/c_api_types.h" 28 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/decode_jpeg_status.h" 29 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/jpeg_common.h" 30 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/jpeg_decompress_buffered_struct.h" 31 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/libc_handle.h" 32 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/libjpeg.h" 33 #include "tensorflow/lite/experimental/acceleration/mini_benchmark/libjpeg_handle.h" 34 #include "tensorflow/lite/string_type.h" 35 #include "tensorflow/lite/string_util.h" 36 37 namespace tflite { 38 namespace acceleration { 39 namespace decode_jpeg_kernel { 40 41 // Extracts the expected size of `jpeg_decompress_struct` from the "struct 42 // mismatch" error message and stores it in `expected_size`. Returns status code 43 // kTfLiteOk if the extraction was successful, error otherwise. 44 Status ExtractSizeFromErrorMessage(const std::string& error_message, 45 size_t& expected_size); 46 47 class LibjpegDecoder { 48 public: 49 // The maximum height allowed for the decoded image. Any attempt to call 50 // DecodeImage for an image with height or width over the allowed limits will 51 // fail. 52 // The size is define to 10,000 lines. 53 static const size_t kMaxImageHeight; 54 // The maximum width allowed for the decoded image. Any attempt to call 55 // DecodeImage for an image with height or width over the allowed limits will 56 // fail. 57 // The size is define to 10,000 pixels per line. 58 static const size_t kMaxImageWidth; 59 60 // Creates and initialises the decoder. 61 // Dynamically loads libjpeg (into handle_) and sets the expected size for 62 // `jpeg_decompress_struct` (in expected_size_for_decompress_struct_). Returns 63 // an initalised instance of decoder if successful, else returns nullptr. 64 // Stores initialisation status in status. 65 static std::unique_ptr<LibjpegDecoder> Create(Status& status); 66 Status DecodeImage(const tflite::StringRef& encoded, 67 const JpegHeader& expected_image_dimensions, 68 unsigned char* decoded, const size_t& decoded_size) const; 69 70 private: LibjpegDecoder(LibCHandle libc_handle)71 explicit LibjpegDecoder(LibCHandle libc_handle) 72 : libc_handle_(std::move(libc_handle)) {} 73 // Wraps all objects required for using the libjpeg library. 74 // This is to avoid stack-allocating these variables in the function that 75 // calls setjmp(). 76 class Impl { 77 public: 78 explicit Impl(size_t decompress_struct_size, const LibjpegHandle* handle); ~Impl()79 ~Impl() { jpeg_destroy_decompress(); } 80 Impl(const Impl&) = delete; 81 Impl& operator=(const Impl&) = delete; 82 Impl(Impl&& other) = delete; 83 Impl& operator=(Impl&& other) = delete; 84 85 // Wrapping calls to LibjpegHandle functions in Run and RunAndSetStatus. jpeg_CreateDecompress(int version,size_t struct_size)86 TfLiteStatus jpeg_CreateDecompress(int version, size_t struct_size) { 87 // Note: It is safe to call jpeg_destroy_decompress even if the 88 // corresponding call to create_jpeg_decompress fails. See the end of 89 // section "Compression details" in 90 // https://www.freedesktop.org/wiki/Software/libjpeg/. 91 safe_to_invoke_destroy_decompress_ = true; 92 return Run(&LibjpegHandle::jpeg_create_decompress_, version, struct_size); 93 } jpeg_stdio_src(FILE * infile)94 TfLiteStatus jpeg_stdio_src(FILE* infile) { 95 return Run(&LibjpegHandle::jpeg_stdio_src_, infile); 96 } 97 jpeg_read_header(int & read_header_result,boolean require_image)98 TfLiteStatus jpeg_read_header(int& read_header_result, 99 boolean require_image) { 100 return RunAndSetResult(&LibjpegHandle::jpeg_read_header_, 101 &read_header_result, require_image); 102 } 103 jpeg_start_decompress(boolean & start_decompress_result)104 TfLiteStatus jpeg_start_decompress(boolean& start_decompress_result) { 105 return RunAndSetResult(&LibjpegHandle::jpeg_start_decompress_, 106 &start_decompress_result); 107 } jpeg_read_scanlines(unsigned int & read_scanlines_result,JSAMPARRAY scanlines,JDIMENSION max_lines)108 TfLiteStatus jpeg_read_scanlines(unsigned int& read_scanlines_result, 109 JSAMPARRAY scanlines, 110 JDIMENSION max_lines) { 111 return RunAndSetResult(&LibjpegHandle::jpeg_read_scanlines_, 112 &read_scanlines_result, scanlines, max_lines); 113 } jpeg_finish_decompress(boolean & finish_decompress_result)114 TfLiteStatus jpeg_finish_decompress(boolean& finish_decompress_result) { 115 return RunAndSetResult(&LibjpegHandle::jpeg_finish_decompress_, 116 &finish_decompress_result); 117 } jpeg_destroy_decompress()118 TfLiteStatus jpeg_destroy_decompress() { 119 if (safe_to_invoke_destroy_decompress_) { 120 safe_to_invoke_destroy_decompress_ = false; 121 return Run(&LibjpegHandle::jpeg_destroy_decompress_); 122 } 123 return kTfLiteOk; 124 } 125 126 // Status from the libjpeg layer that is to be returned to the caller. status()127 Status status() { return status_; } 128 129 private: 130 // Delegates to one of the LibjpegHandle::jpeg_* methods. 131 // This is to restrict the call to setjmp() to a stack frame free from 132 // stack allocated C++ variables. The type of f is T 133 // (LibjpegHandle::*f)(Args...), for some T. All args must be 134 // pointers/references/primitive types. Since we use a 135 // non-suspending JPEG encoded data source, return value from a 136 // LibjpegHandle::jpeg_* methods is not required by client and hence 137 // discarded. Returns an Ok status if the execution was successful, error 138 // otherwise. 139 template <typename Fn, typename... Args> Run(Fn f,Args...args)140 TfLiteStatus Run(Fn f, Args... args) { 141 // Note(1): C++ variables local to this function should not be stack 142 // allocated 143 // and should be passed as pointers or references. Using setjmp/longjmp 144 // with stack allocated C++ objects that have non-trivial destructors can 145 // lead to undefined behaviour. 146 // https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=88046492 147 // All such variables whose scope contains calls to libjpeg (that may do a 148 // longjmp) should be passed in as arguments. 149 // 150 // Note(2): All other variables local to this function that need to be 151 // accessed after longjmp() returns control to this function, should be 152 // volatile-qualified. 153 // After invoking longjump(), non-volatile local variables should not be 154 // accessed for two reasons: 155 // - their values may be indeterminate. According to the C standard, if 156 // the variable's value has changed between setjmp() and longjmp(), their 157 // value is considered indeterminate, and accessing them is undefined 158 // behaviour. 159 // https://wiki.sei.cmu.edu/confluence/display/c/MSC22-C.+Use+the+setjmp%28%29%2C+longjmp%28%29+facility+securely 160 // - the register storing such variables might be clobbered. Even if the 161 // variable remains unchanged between setjmp() and longjmp(), the stack 162 // slot for the variable may get incorrectly clobbered. This is a known 163 // LLVM bug: https://bugs.llvm.org/show_bug.cgi?id=21183 164 if (setjmp(env_)) return kTfLiteError; 165 (handle_->*f)(cinfo_.get(), args...); 166 return kTfLiteOk; 167 } 168 // Extension of the Run method for non-void JPEG calls when we need to 169 // collect the returned value. 170 // See Run comments above for details. 171 template < 172 typename Fn, typename... Args, 173 typename ResultType = typename std::result_of_t<Fn>, 174 typename = typename std::enable_if<!std::is_void<ResultType>::value> > RunAndSetResult(Fn f,ResultType * result,Args...args)175 TfLiteStatus RunAndSetResult(Fn f, ResultType* result, Args... args) { 176 if (setjmp(env_)) return kTfLiteError; 177 *result = (handle_->*f)(cinfo_.get(), args...); 178 return kTfLiteOk; 179 } 180 // Size of `jpeg_decompress_struct` as expected by libjpeg library. 181 size_t decompress_struct_size_; 182 const LibjpegHandle* handle_; 183 // Using a buffered struct for `jpeg_decompress_struct` as the size expected 184 // by libjpeg can be different from the size of the compiled struct. See 185 // go/libjpeg-android. Note: Since we resize the struct, accessing some of 186 // the fields of this struct may lead to undefined behaviour. For 187 // decompression, only the fields within `jpeg_common_fields` are required 188 // viz. error manager(`err`) and client data(`client_data`). This code 189 // limits its usage to these two fields and we recommend future contributors 190 // to not access fields beyond `jpeg_common_fields`. 191 JpegDecompressBufferedStruct cinfo_; 192 struct jpeg_error_mgr jerr_; 193 // Stores the information of the calling environment which can be restored 194 // later. Libjpeg aborts the program in case of any errors by using longjmp 195 // and then calling exit(). The only way to avoid this, is to transfer the 196 // control flow to the caller by using setjmp/longjmp. 197 jmp_buf env_; 198 static void ErrorExit(j_common_ptr cinfo); 199 // Calls to jpeg_create_decompress and jpeg_destroy_decompress need to be 200 // paired. This flag indicates if it's safe to invoke 201 // jpeg_destroy_decompress. 202 bool safe_to_invoke_destroy_decompress_ = false; 203 // Status of the most recent execution of a LibjpegHandle::jpeg_* method 204 // invoked using Run or RunAndSetResult. 205 Status status_; 206 }; 207 // Size of `jpeg_decompress_struct` as expected by the libjpeg dynamic 208 // library. The expected size is different from the size of the compiled 209 // struct on some Android Devices. See go/libjpeg-android. 210 size_t expected_size_for_decompress_struct_; 211 // Handle to the Libjpeg dynamic library. 212 std::unique_ptr<LibjpegHandle> libjpeg_handle_; 213 // Handle to the LibC dynamic library. 214 LibCHandle libc_handle_; 215 }; 216 217 } // namespace decode_jpeg_kernel 218 } // namespace acceleration 219 } // namespace tflite 220 221 #endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_MINI_BENCHMARK_LIBJPEG_DECODER_H_ 222