1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 16 #ifndef TENSORFLOW_LITE_DELEGATES_GPU_CL_API_H_ 17 #define TENSORFLOW_LITE_DELEGATES_GPU_CL_API_H_ 18 19 #ifdef CL_DELEGATE_NO_GL 20 #define EGL_NO_PROTOTYPES 21 #endif 22 23 #include <EGL/egl.h> 24 25 #include <cstdint> 26 #include <memory> 27 28 #include "absl/types/span.h" 29 #include "tensorflow/lite/delegates/gpu/api.h" 30 #include "tensorflow/lite/delegates/gpu/cl/serialization.h" 31 #include "tensorflow/lite/delegates/gpu/common/model.h" 32 #include "tensorflow/lite/delegates/gpu/common/status.h" 33 34 // Usage example: 35 // 36 // std::unique_ptr<InferenceEnvironment> env; 37 // RETURN_IF_ERROR(NewInferenceEnvironment(option, &env)); 38 // 39 // InferenceOptions options; 40 // 41 // std::unique_ptr<InferenceBuilder> builder; 42 // RETURN_IF_ERROR(env->NewInferenceBuilder(options, model, &builder)); 43 // // now builder is ready to prepare inference runner. 44 // 45 // ----------------- 46 // Supported formats 47 // ----------------- 48 // 49 // OpenCL implementation uses 2D textures as the primary format. 50 // Tensor in HWDC4 layout is {TEXTURE_2D, RGBA, width := W*D, height := H}. 51 // 52 53 namespace tflite { 54 namespace gpu { 55 namespace cl { 56 57 struct InferenceOptions : public tflite::gpu::InferenceOptions {}; 58 59 // Indicates environment 60 struct InferenceEnvironmentProperties { 61 bool is_opencl_available = false; 62 63 // GL objects (buffers and textures) could be shared with CL context. 64 bool is_gl_sharing_supported = false; 65 66 // Indicates whether fast GL->CL synchronization is supported. 67 bool is_gl_to_cl_fast_sync_supported = false; 68 69 // Indicates whether fast CL->GL synchronization is supported. 70 bool is_cl_to_gl_fast_sync_supported = false; 71 }; 72 73 // Environment manages all resources that need to stay until any inference is 74 // running using OpenCL backend. 75 class InferenceEnvironment { 76 public: ~InferenceEnvironment()77 virtual ~InferenceEnvironment() {} 78 79 // Converts GraphFloat32 into intermediate, device-specific representation. 80 // This serialized_model specific for device and InferenceOptions. 81 // serialized_model cannot be used with another device or InferenceOptions. 82 // Loading serialized_model is much faster than loading GraphFloat32. 83 // serialized_model must be used with appropriate NewInferenceBuilder 84 // method (see below). 85 // Normally BuildSerializedModel method need to be called whenever a model or 86 // OS GPU driver is updated. 87 virtual absl::Status BuildSerializedModel( 88 const InferenceOptions& options, GraphFloat32 model, 89 std::vector<uint8_t>* serialized_model) = 0; 90 91 // Serialized model can became invalid when environment changes. In this case 92 // this call will fail and model must be regenerated(with 93 // BuildSerializedModel). 94 virtual absl::Status NewInferenceBuilder( 95 const absl::Span<const uint8_t> serialized_model, 96 std::unique_ptr<InferenceBuilder>* builder) = 0; 97 98 virtual absl::Status NewInferenceBuilder( 99 const InferenceOptions& options, GraphFloat32 model, 100 std::unique_ptr<InferenceBuilder>* builder) = 0; 101 102 // Returns opaque binary blob that contains a collection of already compiled 103 // OpenCL kernels present in a cache. Returned data could be re-used later 104 // to speed up compilation time when new environment is created for the same 105 // set of models. 106 // Returned data is valid only if used on the same device, otherwise it will 107 // not be compatible and will be discarded. 108 virtual std::vector<uint8_t> GetSerializedBinaryCache() const = 0; 109 }; 110 111 struct InferenceEnvironmentOptions { 112 // If any of these objects are set, created environment will use them instead 113 // of creating/choosing own instances. 114 cl_device_id device = nullptr; 115 cl_context context = nullptr; 116 cl_command_queue command_queue = nullptr; 117 118 // Whenever input and/or output is GL object, EGL display and context must be 119 // set to create GL aware OpenCL context. Do not set these variables whenever 120 // GL interoperability is not needed. 121 // It is the error to set egl_display, egl_context AND context at the same 122 // time. If egl_display and egl_context are set, they will be used to create 123 // GL-aware CL context. 124 EGLDisplay egl_display = EGL_NO_DISPLAY; 125 EGLContext egl_context = EGL_NO_CONTEXT; 126 127 // Should contain data returned from 128 // InferenceEnvironment::GetSerializedBinaryCache method. 129 // Invalid or incompatible data will be discarded. Compiled binary may become 130 // incompatible when GPU driver is updated. 131 absl::Span<const uint8_t> serialized_binary_cache; 132 IsGlAwareInferenceEnvironmentOptions133 bool IsGlAware() const { 134 return egl_context != EGL_NO_CONTEXT && egl_display != EGL_NO_DISPLAY; 135 } 136 }; 137 138 // Creates new OpenCL environment that needs to stay around until all inference 139 // runners are destroyed. 140 absl::Status NewInferenceEnvironment( 141 const InferenceEnvironmentOptions& options, 142 std::unique_ptr<InferenceEnvironment>* environment, 143 InferenceEnvironmentProperties* properties /* optional */); 144 145 class CLInferenceRunner : public ::tflite::gpu::InferenceRunner { 146 public: 147 // The RunWithoutExternalBufferCopy provides a contract where the user of this 148 // interface does not need 149 // a. Inputs to be copied to the internal GPU buffer from the external CPU 150 // input buffer 151 // b. Outputs to be copied from the internal GPU buffer to the 152 // external CPU buffer 153 // 154 // The user of this interface is responsible for copying the inputs prior to 155 // running the GPU kernels and outputs post running with the other interfaces 156 // provided here. 157 virtual absl::Status RunWithoutExternalBufferCopy() = 0; 158 159 // Copies from the external input tensor (normally CPU buffer) to the internal 160 // OpenCL buffer. This call blocks until the copy is finished. 161 virtual absl::Status CopyFromExternalInput(int index) = 0; 162 163 // Copies from the internal output OpenCL buffer to the external output 164 // tensor. This call blocks until the copy is finished. 165 virtual absl::Status CopyToExternalOutput(int index) = 0; 166 }; 167 168 } // namespace cl 169 } // namespace gpu 170 } // namespace tflite 171 172 #endif // TENSORFLOW_LITE_DELEGATES_GPU_CL_API_H_ 173