1 /**
2 * Copyright 2020 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "src/inner_context.h"
17 #include <algorithm>
18 #include "include/errorcode.h"
19 #include "src/common/log_adapter.h"
20 #include "src/common/log_util.h"
21 #ifdef ENABLE_MINDRT
22 #include "thread/actor_threadpool.h"
23 #endif
24 #ifdef SUPPORT_NPU
25 #include "include/HiAiModelManagerType.h"
26 #endif
27 #ifdef GPU_OPENCL
28 #include "src/runtime/gpu/opencl/opencl_runtime.h"
29 #endif
30
31 namespace mindspore::lite {
32 namespace {
33 #ifdef ENABLE_MINDRT
34 constexpr int kDefaultParallelNum = 2;
35 #endif
36 const constexpr int kMaxLiteContextDeviceNums = 2;
37 const constexpr int kMaxInnerContextDeviceNums = 3;
38 } // namespace
39
InitDeviceFp16()40 void InnerContext::InitDeviceFp16() {
41 #if defined(ENABLE_ARM) && defined(ENABLE_FP16)
42 CpuInfo cpu_info;
43 device_and_pkg_support_fp16_ = cpu_info.ArmIsSupportFp16();
44 #else
45 device_and_pkg_support_fp16_ = false;
46 #endif
47 }
48
InnerContext(const Context * context)49 InnerContext::InnerContext(const Context *context) {
50 if (context != nullptr) {
51 this->allocator = context->allocator;
52 this->thread_num_ = context->thread_num_;
53 this->enable_parallel_ = context->enable_parallel_;
54 this->affinity_core_list_ = context->affinity_core_list_;
55 SetContextDevice(context);
56 this->delegate = context->delegate;
57 }
58 InitDeviceFp16();
59 }
60
SetContextDevice(const Context * context)61 void InnerContext::SetContextDevice(const Context *context) {
62 this->device_list_.clear();
63
64 if (context->device_list_.size() > kMaxLiteContextDeviceNums || context->device_list_.size() <= 0) {
65 return;
66 }
67 if (context->device_list_.front().device_type_ != DT_CPU) {
68 return;
69 }
70
71 /* user set order for different device */
72 if (context->device_list_.size() < kMaxLiteContextDeviceNums) {
73 this->device_list_.push_back(context->device_list_.front());
74 return;
75 }
76
77 /* keep compatibility :
78 * if user set CPU & NPU/GPU
79 * NPU/GPU higher priority */
80 bool isUserSetNPU = context->device_list_.end() !=
81 std::find_if(this->device_list_.begin(), this->device_list_.end(),
82 [](const DeviceContext &device) { return device.device_type_ == DT_NPU; });
83 bool isUserSetGPU = context->device_list_.end() !=
84 std::find_if(this->device_list_.begin(), this->device_list_.end(),
85 [](const DeviceContext &device) { return device.device_type_ == DT_GPU; });
86 bool isUserSetNNRt = context->device_list_.end() !=
87 std::find_if(this->device_list_.begin(), this->device_list_.end(),
88 [](const DeviceContext &device) { return device.device_type_ == DT_NNRT; });
89 if (isUserSetGPU == false && isUserSetNPU == false && isUserSetNNRt == false) {
90 return;
91 }
92
93 /* add GPU/NPU first */
94 for (auto &device_ctx : context->device_list_) {
95 if (device_ctx.device_type_ != DT_CPU) {
96 this->device_list_.push_back(device_ctx);
97 }
98 }
99
100 /* add CPU */
101 for (auto &device_ctx : context->device_list_) {
102 if (device_ctx.device_type_ == DT_CPU) {
103 if (isUserSetNPU || isUserSetNNRt || (isUserSetGPU && enable_parallel_ == false)) {
104 auto cpu_ctx = device_ctx;
105 cpu_ctx.device_info_.cpu_device_info_.cpu_bind_mode_ = NO_BIND;
106 this->device_list_.push_back(cpu_ctx);
107 } else {
108 this->device_list_.push_back(device_ctx);
109 }
110 }
111 }
112 return;
113 }
114
Init()115 int InnerContext::Init() {
116 if (RET_OK != this->IsValid()) {
117 MS_LOG(ERROR) << "Context is not valid";
118 return RET_NOT_SUPPORT;
119 }
120 if (this->thread_pool_ == nullptr) {
121 BindMode bind_mode = Power_NoBind;
122 if (this->IsCpuEnabled()) {
123 bind_mode = static_cast<BindMode>(this->GetCpuDeviceInfo()->cpu_bind_mode_);
124 }
125
126 #ifdef ENABLE_MINDRT
127 int actor_parallel_thread = this->enable_parallel_ ? kDefaultParallelNum : 1;
128 if (this->affinity_core_list_.empty()) {
129 thread_pool_ = ActorThreadPool::CreateThreadPool(actor_parallel_thread, this->thread_num_, bind_mode);
130 if (thread_pool_ == nullptr) {
131 MS_LOG(ERROR) << "Create ThreadPool failed";
132 return RET_NULL_PTR;
133 }
134 } else {
135 thread_pool_ =
136 ActorThreadPool::CreateThreadPool(actor_parallel_thread, this->thread_num_, this->affinity_core_list_);
137 if (thread_pool_ == nullptr) {
138 MS_LOG(ERROR) << "Create ThreadPool failed";
139 return RET_NULL_PTR;
140 }
141 }
142 #else
143 thread_pool_ = ThreadPool::CreateThreadPool(thread_num_ - 1);
144 thread_pool_->SetCpuAffinity(static_cast<mindspore::BindMode>(bind_mode));
145 #endif
146 }
147
148 if (this->allocator == nullptr) {
149 this->allocator = mindspore::Allocator::Create();
150 if (this->allocator == nullptr) {
151 MS_LOG(ERROR) << "Create Allocator failed";
152 return RET_NULL_PTR;
153 }
154 }
155 if (IsNpuEnabled()) {
156 MS_LOG(DEBUG) << "NPU enabled.";
157 #ifdef SUPPORT_NPU
158 for (auto &device_ctx : this->device_list_) {
159 if (device_ctx.device_type_ == DT_NPU &&
160 device_ctx.device_info_.npu_device_info_.frequency_ != hiai::AiModelDescription_Frequency_LOW &&
161 device_ctx.device_info_.npu_device_info_.frequency_ != hiai::AiModelDescription_Frequency_MEDIUM &&
162 device_ctx.device_info_.npu_device_info_.frequency_ != hiai::AiModelDescription_Frequency_HIGH &&
163 device_ctx.device_info_.npu_device_info_.frequency_ != hiai::AiModelDescription_Frequency_EXTREME) {
164 MS_LOG(INFO) << "NPU frequency set to 3, original value "
165 << device_ctx.device_info_.npu_device_info_.frequency_;
166 device_ctx.device_info_.npu_device_info_.frequency_ = hiai::AiModelDescription_Frequency_HIGH;
167 }
168 }
169 #endif
170 }
171 if (IsGpuEnabled()) {
172 MS_LOG(DEBUG) << "GPU enabled.";
173 }
174 #ifdef SUPPORT_NNRT
175 if (IsNNRtEnabled()) {
176 MS_LOG(DEBUG) << "NNRt enabled.";
177 }
178 #endif
179 return RET_OK;
180 }
181
~InnerContext()182 InnerContext::~InnerContext() {
183 if (this->thread_pool_ != nullptr) {
184 delete thread_pool_;
185 this->thread_pool_ = nullptr;
186 }
187 }
188
IsValid() const189 int InnerContext::IsValid() const {
190 if (this->device_list_.empty()) {
191 MS_LOG(ERROR) << "Device list is empty.";
192 return RET_NOT_SUPPORT;
193 }
194 if (this->device_list_.size() > kMaxInnerContextDeviceNums) {
195 MS_LOG(ERROR) << "Not support device list more than 2.";
196 return RET_NOT_SUPPORT;
197 }
198 if (thread_num_ < 1) {
199 MS_LOG(ERROR) << "Thread num smaller than 1 is not allowed.";
200 return RET_NOT_SUPPORT;
201 }
202 if (!IsAllDeviceTypeValid()) {
203 MS_LOG(ERROR) << "Device type should be one of DT_CPU, DT_GPU or DT_NPU.";
204 return RET_NOT_SUPPORT;
205 }
206
207 if (IsCpuBindModeInvalid()) {
208 MS_LOG(ERROR) << "CPU bind mode should be one of NO_BIND, HIGHER_CPU or MID_CPU.";
209 return RET_NOT_SUPPORT;
210 }
211
212 #ifndef SUPPORT_GPU
213 if (IsUserSetGpu()) {
214 MS_LOG(ERROR) << "GPU is not supported.";
215 return RET_NOT_SUPPORT;
216 }
217 #endif
218
219 #ifndef SUPPORT_NPU
220 if (IsUserSetNpu()) {
221 MS_LOG(ERROR) << "NPU is not supported.";
222 return RET_NOT_SUPPORT;
223 }
224 #endif
225
226 #ifndef SUPPORT_NNRT
227 if (IsUserSetNNRt()) {
228 MS_LOG(ERROR) << "NNRt is not supported.";
229 return RET_NOT_SUPPORT;
230 }
231 #endif
232 return RET_OK;
233 }
234
IsCpuFloat16Enabled() const235 bool InnerContext::IsCpuFloat16Enabled() const {
236 if (!IsCpuEnabled()) {
237 return false;
238 }
239 if (!device_and_pkg_support_fp16_) {
240 return false;
241 }
242 return GetCpuInfo().enable_float16_;
243 }
244
IsGpuFloat16Enabled() const245 bool InnerContext::IsGpuFloat16Enabled() const {
246 #ifdef GPU_OPENCL
247 if (!IsGpuEnabled()) {
248 return false;
249 }
250 opencl::OpenCLRuntimeInnerWrapper wrapper;
251 if (!wrapper.GetInstance()->GetFp16Enable()) {
252 return false;
253 }
254 return GetGpuInfo().enable_float16_;
255 #else
256 return false;
257 #endif
258 }
259
IsCpuEnabled() const260 bool InnerContext::IsCpuEnabled() const { return IsUserSetCpu(); }
261
GetCpuDeviceInfo() const262 const CpuDeviceInfo *InnerContext::GetCpuDeviceInfo() const {
263 if (IsUserSetCpu() == false) {
264 return nullptr;
265 }
266 const DeviceInfo *device_info = nullptr;
267
268 (void)std::find_if(this->device_list_.begin(), this->device_list_.end(), [&](const DeviceContext &device) {
269 if (device.device_type_ == DeviceType::DT_CPU) {
270 device_info = &device.device_info_;
271 return true;
272 }
273 return false;
274 });
275
276 return reinterpret_cast<const CpuDeviceInfo *>(device_info);
277 }
278
IsGpuEnabled() const279 bool InnerContext::IsGpuEnabled() const {
280 #ifdef SUPPORT_GPU
281 return IsUserSetGpu();
282 #else
283 return false;
284 #endif
285 }
286
IsNpuEnabled() const287 bool InnerContext::IsNpuEnabled() const {
288 #ifdef SUPPORT_NPU
289 return IsUserSetNpu();
290 #else
291 return false;
292 #endif
293 }
294
IsNNRtEnabled() const295 bool InnerContext::IsNNRtEnabled() const {
296 #ifdef SUPPORT_NNRT
297 return IsUserSetNNRt();
298 #else
299 return false;
300 #endif
301 }
302
IsProviderEnabled() const303 bool InnerContext::IsProviderEnabled() const {
304 return this->device_list_.end() !=
305 std::find_if(this->device_list_.begin(), this->device_list_.end(),
306 [](const DeviceContext &device) { return !device.provider_.empty(); });
307 }
308
IsAllDeviceTypeValid() const309 bool InnerContext::IsAllDeviceTypeValid() const {
310 return std::all_of(this->device_list_.begin(), this->device_list_.end(), [](const DeviceContext &device) {
311 return device.device_type_ >= DT_CPU && device.device_type_ <= DT_NNRT;
312 });
313 }
314
IsCpuBindModeInvalid() const315 bool InnerContext::IsCpuBindModeInvalid() const {
316 return this->device_list_.end() !=
317 std::find_if(this->device_list_.begin(), this->device_list_.end(), [](const DeviceContext &device) {
318 return device.device_type_ == DT_CPU && (device.device_info_.cpu_device_info_.cpu_bind_mode_ < NO_BIND ||
319 device.device_info_.cpu_device_info_.cpu_bind_mode_ > MID_CPU);
320 });
321 }
322
IsUserSetCpu() const323 bool InnerContext::IsUserSetCpu() const {
324 return this->device_list_.end() !=
325 std::find_if(this->device_list_.begin(), this->device_list_.end(),
326 [](const DeviceContext &device) { return device.device_type_ == DT_CPU; });
327 }
328
IsUserSetGpu() const329 bool InnerContext::IsUserSetGpu() const {
330 return this->device_list_.end() !=
331 std::find_if(this->device_list_.begin(), this->device_list_.end(),
332 [](const DeviceContext &device) { return device.device_type_ == DT_GPU; });
333 }
334
IsUserSetNpu() const335 bool InnerContext::IsUserSetNpu() const {
336 return this->device_list_.end() !=
337 std::find_if(this->device_list_.begin(), this->device_list_.end(),
338 [](const DeviceContext &device) { return device.device_type_ == DT_NPU; });
339 }
340
IsUserSetNNRt() const341 bool InnerContext::IsUserSetNNRt() const {
342 return this->device_list_.end() !=
343 std::find_if(this->device_list_.begin(), this->device_list_.end(),
344 [](const DeviceContext &device) { return device.device_type_ == DT_NNRT; });
345 }
346
GetProviders() const347 std::set<std::string> InnerContext::GetProviders() const {
348 std::set<std::string> providers;
349 for (auto &&device : device_list_) {
350 if (!device.provider_.empty()) {
351 providers.insert(device.provider_);
352 }
353 }
354 return providers;
355 }
356
GetCpuInfo() const357 CpuDeviceInfo InnerContext::GetCpuInfo() const {
358 auto iter = std::find_if(this->device_list_.begin(), this->device_list_.end(),
359 [](const DeviceContext &device) { return device.device_type_ == DT_CPU; });
360 if (iter == this->device_list_.end()) {
361 return {};
362 } else {
363 return iter->device_info_.cpu_device_info_;
364 }
365 }
366
GetGpuInfo() const367 GpuDeviceInfo InnerContext::GetGpuInfo() const {
368 auto iter = std::find_if(this->device_list_.begin(), this->device_list_.end(),
369 [](const DeviceContext &device) { return device.device_type_ == DT_GPU; });
370 if (iter == this->device_list_.end()) {
371 return {};
372 } else {
373 return iter->device_info_.gpu_device_info_;
374 }
375 }
376
GetNpuInfo() const377 NpuDeviceInfo InnerContext::GetNpuInfo() const {
378 auto iter = std::find_if(this->device_list_.begin(), this->device_list_.end(),
379 [](const DeviceContext &device) { return device.device_type_ == DT_NPU; });
380 if (iter == this->device_list_.end()) {
381 return {};
382 } else {
383 return iter->device_info_.npu_device_info_;
384 }
385 }
386
thread_pool() const387 ThreadPool *InnerContext::thread_pool() const { return thread_pool_; }
388
device_and_pkg_support_fp16() const389 bool InnerContext::device_and_pkg_support_fp16() const { return this->device_and_pkg_support_fp16_; }
390
ParallelLaunch(const Context * context,const Func & func,Content content,int task_num)391 int ParallelLaunch(const Context *context, const Func &func, Content content, int task_num) {
392 ThreadPool *pool = static_cast<const lite::InnerContext *>(context)->thread_pool();
393 if (pool == nullptr) {
394 MS_LOG(ERROR) << "thread pool is nullptr";
395 return RET_NULL_PTR;
396 }
397 return pool->ParallelLaunch(func, content, task_num);
398 }
399 } // namespace mindspore::lite
400