1 /** 2 * Copyright 2020-2023 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef MINDSPORE_LITE_SRC_RUNTIME_INNER_CONTEXT_H_ 18 #define MINDSPORE_LITE_SRC_RUNTIME_INNER_CONTEXT_H_ 19 #include <set> 20 #include <string> 21 #include <vector> 22 #include <memory> 23 #include <unordered_map> 24 #ifdef BFC_MEMORY 25 #include "src/extendrt/dynamic_mem_allocator.h" 26 #else 27 #include "src/litert/inner_allocator.h" 28 #endif 29 #include "thread/threadpool.h" 30 #include "nnacl/op_base.h" 31 #include "nnacl/kernel.h" 32 #ifdef ENABLE_ARM 33 #include "src/litert/cpu_info.h" 34 #endif 35 #include "include/lite_types.h" 36 #include "src/litert/infer_manager.h" 37 38 namespace mindspore { 39 class DeviceInfoContext; 40 } 41 42 namespace mindspore::lite { 43 typedef struct CpuDeviceInfo { 44 bool enable_float16_ = false; /**< prior enable float16 inference */ 45 CpuBindMode cpu_bind_mode_ = MID_CPU; 46 } CpuDeviceInfo; 47 48 typedef struct GpuDeviceInfo { 49 bool enable_float16_ = false; /**< prior enable float16 inference */ 50 uint32_t gpu_device_id_ = 0; 51 int rank_id_ = 0; 52 int group_size_ = 0; 53 bool enable_gl_texture_ = false; /**<enable sharing OpenGL texture with OpenCL */ 54 void *gl_context_ = nullptr; 55 void *gl_display_ = nullptr; 56 } GpuDeviceInfo; 57 58 typedef struct NpuDeviceInfo { 59 bool enable_float16_ = false; /**< prior enable float16 inference */ 60 int frequency_ = 3; /**< npu frequency inference, low 1, medium 2, high 3, extreme 4, other values will be set to 3 */ 61 } NpuDeviceInfo; 62 63 typedef struct AscendDeviceInfo { 64 uint32_t device_id_ = 0; 65 std::string batch_size_; 66 std::string image_size_; 67 } AscendDeviceInfo; 68 69 /// \brief CustomDeviceInfo defined for user defined device configuration information. 70 typedef struct CustomDeviceInfo { 71 std::shared_ptr<DeviceInfoContext> user_defined_device_info_; 72 } CustomDeviceInfo; 73 74 typedef struct Extension { 75 std::string name; // config name 76 std::vector<uint8_t> value; // config value 77 } Extension; 78 79 typedef struct NNRtDeviceInfo { 80 size_t device_id_ = 0; 81 int priority_ = 0; 82 int performance_mode_ = 0; 83 bool enable_fp16_ = false; 84 std::vector<Extension> extensions_; 85 } NNRtDeviceInfo; 86 87 struct DeviceInfo { 88 CpuDeviceInfo cpu_device_info_; 89 GpuDeviceInfo gpu_device_info_; 90 NpuDeviceInfo npu_device_info_; 91 AscendDeviceInfo ascend_device_info_; 92 CustomDeviceInfo custom_device_info_; 93 NNRtDeviceInfo nnrt_device_info_; 94 }; 95 96 struct DeviceContext { 97 DeviceType device_type_ = DT_CPU; 98 DeviceInfo device_info_; 99 std::string provider_{}; 100 std::string provider_device_{}; 101 AllocatorPtr allocator_ = nullptr; 102 }; 103 104 typedef struct InstructionsContext { 105 // Instructions should be checked in the beginning. 106 bool support_fp16 = false; 107 bool support_sdot = false; 108 bool support_sse = false; 109 bool support_avx512 = false; 110 } InstructionsContext; 111 112 struct MS_API InnerContext { 113 public: 114 InnerContext(); 115 virtual ~InnerContext(); 116 int Init(); 117 bool IsCpuFloat16Enabled() const; 118 bool IsGpuFloat16Enabled() const; 119 bool IsNpuFloat16Enabled() const; 120 bool IsGLTextureEnabled() const; 121 bool IsDeviceTypeEnabled(DeviceType type) const; 122 bool IsProviderEnabled() const; 123 int GetDelegateMode() const; 124 std::set<std::string> GetProviders() const; 125 DeviceInfo GetDeviceInfo(DeviceType type) const; 126 std::set<void *> GetLinkInfo(void *pre) const; 127 std::unordered_map<void *, std::set<void *>> GetAllLinkInfo() const; 128 void SetLinkInfo(void *pre, void *suc); 129 void SetAllLinkInfo(const std::unordered_map<void *, std::set<void *>> &all_link_info); 130 void ReplaceLinkInfoReceiverWithNewOne(void *new_receiver, void *old_receiver); 131 void ReplaceLinkInfoSenderWithNewOne(void *new_sender, void *old_sender); SetBindRunnerIdInnerContext132 inline void SetBindRunnerId(std::string runner_id) { runner_id_ = runner_id; } set_infer_checkerInnerContext133 inline void set_infer_checker(const InferChecker checker) { infer_checker_ = checker; } get_infer_checkerInnerContext134 inline const InferChecker get_infer_checker() const { return infer_checker_; } set_schema_versionInnerContext135 inline void set_schema_version(const int schema_version) { this->schema_version_ = schema_version; } get_schema_versionInnerContext136 inline const int &get_schema_version() const { return schema_version_; } 137 int CreateThreadPool(bool is_control_flow); 138 void DeleteThreadPool(); 139 140 std::string vendor_name_; 141 InstructionsContext instructions_ctx_; 142 int thread_num_ = 2; /**< thread number config for thread pool */ 143 int inter_op_parallel_num_ = 1; 144 bool enable_parallel_ = false; 145 std::vector<int> affinity_core_list_; /**< explicitly specify the core to be bound. priority use affinity core list */ 146 AllocatorPtr allocator = nullptr; 147 std::vector<DeviceContext> device_list_ = {{DT_CPU, {{false, MID_CPU}}}}; 148 int delegate_mode_ = 0; 149 DelegatePtr delegate = nullptr; 150 bool float_mode = false; /**< convert full quant model to float model */ 151 152 bool device_and_pkg_support_fp16_ = false; 153 ThreadPool *thread_pool_ = nullptr; 154 InferChecker infer_checker_{InferCheckerOutput}; 155 // key is the precursor tensor's pointer, value is the group of successors' pointer. 156 std::unordered_map<void *, std::set<void *>> link_info_{}; GetExecEnvInnerContext157 const ExecEnv *GetExecEnv() const { return &exec_env_; } 158 159 private: 160 int IsValid(); 161 bool IsAllDeviceTypeValid() const; 162 bool IsCpuBindModeInvalid() const; 163 void InitExecEnv(); 164 165 std::string runner_id_; 166 BindMode bind_mode_{Power_NoBind}; 167 size_t actor_thread_num_{0}; 168 ExecEnv exec_env_; 169 int schema_version_{SCHEMA_VERSION::SCHEMA_CUR}; 170 }; 171 172 int ParallelLaunch(const InnerContext *context, const Func &func, Content content, int task_num); 173 } // namespace mindspore::lite 174 175 #endif // MINDSPORE_LITE_SRC_RUNTIME_INNER_CONTEXT_H_ 176