1 /**
2 * Copyright 2020-2023 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_EXEC_H_
18 #define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_EXEC_H_
19 #include <string>
20 #include <vector>
21 #include <memory>
22 #include <utility>
23 #include <algorithm>
24 #include "src/common/utils.h"
25 #include "src/common/log_util.h"
26 #ifdef ENABLE_ARM
27 #include <arm_neon.h>
28 #endif
29 #include "nnacl/op_base.h"
30 #include "src/litert/inner_context.h"
31 #include "src/tensor.h"
32 #include "include/errorcode.h"
33 #include "include/api/kernel.h"
34 #include "src/litert/cxx_api/tensor/tensor_impl.h"
35 #include "src/litert/lite_kernel.h"
36 #include "include/api/delegate.h"
37 #include "extendrt/mindir_loader/abstract_kernel.h"
38 #include "include/lite_types.h"
39 #include "src/infer/primitive_type.h"
40 #include "src/infer/tensor.h"
41 #include "src/infer/context.h"
42
43 namespace mindspore::lite {
44 using KernelCallBack = std::function<bool(std::vector<lite::Tensor *> inputs, std::vector<lite::Tensor *> outputs,
45 const MSCallBackParam &opInfo)>;
46 }
47
48 using mindspore::infer::Abstractkernel;
49 using mindspore::lite::KernelCallBack;
50
51 namespace mindspore::kernel {
52 enum KERNEL_ARCH { kCPU, kGPU, kAPU, kNPU, kCustom, kDelegate, kACL, kKernelArch_MIN = kCPU, kKernelArch_MAX = kAPU };
53 static const char *const kBuiltin = "Builtin";
54
55 struct KernelKey {
56 KERNEL_ARCH arch = kCPU;
57 TypeId data_type = kTypeUnknown;
58 Format format = Format::NHWC;
59 int type = 0;
60 std::string kernel_arch;
61 std::string provider{kBuiltin};
62
63 bool operator==(const KernelKey &dst) const {
64 return type == dst.type && kernel_arch == dst.kernel_arch && provider == dst.provider && arch == dst.arch &&
65 data_type == dst.data_type && format == dst.format;
66 }
67 };
68
69 enum SubGraphType {
70 kNotSubGraph = 0,
71 kCpuFP32SubGraph,
72 kCpuFP16SubGraph,
73 kGpuFp32SubGraph,
74 kGpuFp16SubGraph,
75 kNpuSubGraph,
76 kApuSubGraph,
77 kCustomSubGraph,
78 kEntranceSubGraph,
79 kExitSubGraph,
80 kStackSubGraph,
81 kAclSubGraph
82 };
83
84 class KernelExec {
85 public:
KernelExec()86 KernelExec() {
87 this->in_kernels_.clear();
88 this->out_kernels_.clear();
89 }
90
KernelExec(std::shared_ptr<MSKernel> kernel)91 explicit KernelExec(std::shared_ptr<MSKernel> kernel) : kernel_(std::move(kernel)) {
92 this->in_kernels_.clear();
93 this->out_kernels_.clear();
94 }
95
96 virtual ~KernelExec() = default;
97
Execute()98 virtual int Execute() { return DoExecute(); }
99
Execute(const KernelCallBack & before,const KernelCallBack & after)100 virtual int Execute(const KernelCallBack &before, const KernelCallBack &after) {
101 if (before != nullptr) {
102 if (!before(this->in_tensors(), this->out_tensors(), {this->name(), TypeName(type())})) {
103 MS_LOG(WARNING) << "run kernel before_callback failed, name: " << this->name();
104 }
105 }
106
107 auto ret = DoExecute();
108
109 if (after != nullptr) {
110 if (!after(this->in_tensors(), this->out_tensors(), {this->name(), TypeName(type())})) {
111 MS_LOG(WARNING) << "run kernel after_callback failed, name: " << this->name();
112 }
113 }
114 return ret;
115 }
116
117 // called while compiling graph
Prepare()118 virtual int Prepare() {
119 MS_ASSERT(kernel_ != nullptr);
120 return kernel_->Prepare();
121 }
122
IsBuiltin()123 virtual bool IsBuiltin() { return desc_.provider == kBuiltin; }
124
InferShape()125 virtual int InferShape() { return kernel_->InferShape(); }
126
ReSize()127 virtual int ReSize() {
128 MS_ASSERT(kernel_ != nullptr);
129 return kernel_->ReSize();
130 }
131
op_parameter()132 virtual OpParameter *op_parameter() const {
133 MS_ASSERT(kernel_ != nullptr);
134 if (desc_.provider == kBuiltin) {
135 return std::static_pointer_cast<LiteKernel>(kernel_)->op_parameter();
136 }
137 return nullptr;
138 }
139
name()140 std::string name() const {
141 MS_ASSERT(kernel_ != nullptr);
142 return kernel_->name();
143 }
144
set_name(const std::string & name)145 void set_name(const std::string &name) {
146 MS_ASSERT(kernel_ != nullptr);
147 kernel_->set_name(name);
148 }
149
Train()150 virtual int Train() {
151 MS_ASSERT(kernel_ != nullptr);
152 if (desc_.provider == kBuiltin) {
153 return std::static_pointer_cast<Abstractkernel>(kernel_)->Train();
154 }
155 return mindspore::lite::RET_OK;
156 }
157
IsTrain()158 virtual bool IsTrain() const {
159 MS_ASSERT(kernel_ != nullptr);
160 if (desc_.provider == kBuiltin) {
161 return std::static_pointer_cast<Abstractkernel>(kernel_)->IsTrain();
162 }
163 return false;
164 }
165
Eval()166 virtual int Eval() {
167 MS_ASSERT(kernel_ != nullptr);
168 if (desc_.provider == kBuiltin) {
169 return std::static_pointer_cast<Abstractkernel>(kernel_)->Eval();
170 }
171 return mindspore::lite::RET_OK;
172 }
173
IsEval()174 virtual bool IsEval() const {
175 MS_ASSERT(kernel_ != nullptr);
176 if (desc_.provider == kBuiltin) {
177 return std::static_pointer_cast<Abstractkernel>(kernel_)->IsEval();
178 }
179 return false;
180 }
181
182 virtual void SetTrainable(bool trainable = true) {
183 MS_ASSERT(kernel_ != nullptr);
184 if (desc_.provider == kBuiltin) {
185 std::static_pointer_cast<Abstractkernel>(kernel_)->SetTrainable(trainable);
186 }
187 }
188
IsTrainable()189 virtual bool IsTrainable() const {
190 MS_ASSERT(kernel_ != nullptr);
191 if (desc_.provider == kBuiltin) {
192 return std::static_pointer_cast<Abstractkernel>(kernel_)->IsTrainable();
193 }
194 return false;
195 }
196
197 int DoExecute();
198
set_is_model_output(bool is_model_output)199 void set_is_model_output(bool is_model_output) { this->is_model_output_ = is_model_output; }
200
is_model_output()201 bool is_model_output() const { return this->is_model_output_; }
202
InferShapeDone()203 bool InferShapeDone() const {
204 auto checker = context_ != nullptr ? context_->get_infer_checker() : lite::InferCheckerOutput;
205 return checker != nullptr && checker(in_tensors(), out_tensors());
206 }
207
type()208 virtual PrimitiveType type() const {
209 MS_ASSERT(kernel_ != nullptr);
210 return PrimitiveType(std::static_pointer_cast<Abstractkernel>(kernel_)->type());
211 }
212
set_in_tensors(const std::vector<lite::Tensor * > & in_tensors)213 virtual void set_in_tensors(const std::vector<lite::Tensor *> &in_tensors) {
214 MS_ASSERT(kernel_ != nullptr);
215 if (desc_.provider == kBuiltin) {
216 std::static_pointer_cast<Abstractkernel>(kernel_)->set_in_tensors(in_tensors);
217 } else {
218 std::vector<MSTensor> tensors_in;
219 (void)std::transform(in_tensors.begin(), in_tensors.end(), std::back_inserter(tensors_in),
220 [](lite::Tensor *tensor) {
221 auto impl = std::make_shared<mindspore::LiteTensorImpl>(tensor);
222 return mindspore::MSTensor(impl);
223 });
224 kernel_->set_inputs(tensors_in);
225 }
226 }
227
set_in_tensor(lite::Tensor * in_tensor,size_t index)228 virtual void set_in_tensor(lite::Tensor *in_tensor, size_t index) {
229 MS_ASSERT(kernel_ != nullptr);
230 if (desc_.provider == kBuiltin) {
231 std::static_pointer_cast<Abstractkernel>(kernel_)->set_in_tensor(in_tensor, index);
232 } else {
233 MS_ASSERT(index < kernel_->inputs().size());
234 auto impl = std::make_shared<mindspore::LiteTensorImpl>(in_tensor);
235 auto tensor_in = mindspore::MSTensor(impl);
236 kernel_->set_input(tensor_in, static_cast<int>(index));
237 }
238 }
239
set_out_tensors(const std::vector<lite::Tensor * > & out_tensors)240 virtual void set_out_tensors(const std::vector<lite::Tensor *> &out_tensors) {
241 MS_ASSERT(kernel_ != nullptr);
242 if (desc_.provider == kBuiltin) {
243 std::static_pointer_cast<Abstractkernel>(kernel_)->set_out_tensors(out_tensors);
244 } else {
245 std::vector<MSTensor> tensors_out;
246 (void)std::transform(out_tensors.begin(), out_tensors.end(), std::back_inserter(tensors_out),
247 [](lite::Tensor *tensor) {
248 auto impl = std::make_shared<mindspore::LiteTensorImpl>(tensor);
249 return mindspore::MSTensor(impl);
250 });
251 kernel_->set_outputs(tensors_out);
252 }
253 }
254
set_out_tensor(lite::Tensor * out_tensor,size_t index)255 virtual void set_out_tensor(lite::Tensor *out_tensor, size_t index) {
256 MS_ASSERT(kernel_ != nullptr);
257 if (desc_.provider == kBuiltin) {
258 std::static_pointer_cast<Abstractkernel>(kernel_)->set_out_tensor(out_tensor, index);
259 } else {
260 MS_ASSERT(index < kernel_->outputs().size());
261 auto impl = std::make_shared<mindspore::LiteTensorImpl>(out_tensor);
262 auto tensor_out = mindspore::MSTensor(impl);
263 kernel_->set_output(tensor_out, static_cast<int>(index));
264 }
265 }
266
in_tensors()267 virtual const std::vector<lite::Tensor *> &in_tensors() const {
268 MS_ASSERT(kernel_ != nullptr);
269 if (desc_.provider == kBuiltin) {
270 return std::static_pointer_cast<Abstractkernel>(kernel_)->in_tensors();
271 } else {
272 auto &ms_tensors = kernel_->inputs();
273 mutable_in_tensors_.resize(ms_tensors.size());
274 (void)std::transform(ms_tensors.begin(), ms_tensors.end(), mutable_in_tensors_.begin(),
275 [](const mindspore::MSTensor &tensor) {
276 if (tensor.impl() == nullptr) {
277 MS_LOG(ERROR) << "Tensor " << tensor.Name() << " is nullptr.";
278 return static_cast<lite::Tensor *>(nullptr);
279 }
280 auto lite_impl = std::static_pointer_cast<LiteTensorImpl>(tensor.impl());
281 return static_cast<lite::Tensor *>(lite_impl->lite_tensor());
282 });
283 return mutable_in_tensors_;
284 }
285 }
286
out_tensors()287 virtual const std::vector<lite::Tensor *> &out_tensors() const {
288 MS_ASSERT(kernel_ != nullptr);
289 if (desc_.provider == kBuiltin) {
290 return std::static_pointer_cast<Abstractkernel>(kernel_)->out_tensors();
291 } else {
292 auto &ms_tensors = kernel_->outputs();
293 mutable_out_tensors_.resize(ms_tensors.size());
294 (void)std::transform(ms_tensors.begin(), ms_tensors.end(), mutable_out_tensors_.begin(),
295 [](const mindspore::MSTensor &tensor) {
296 if (tensor.impl() == nullptr) {
297 MS_LOG(ERROR) << "Tensor " << tensor.Name() << " is nullptr.";
298 return static_cast<lite::Tensor *>(nullptr);
299 }
300 auto lite_impl = std::static_pointer_cast<LiteTensorImpl>(tensor.impl());
301 return static_cast<lite::Tensor *>(lite_impl->lite_tensor());
302 });
303 return mutable_out_tensors_;
304 }
305 }
306
AddInKernel(KernelExec * kernel)307 void AddInKernel(KernelExec *kernel) {
308 if (!lite::IsContain(this->in_kernels_, kernel)) {
309 this->in_kernels_.emplace_back(kernel);
310 }
311 }
312
AddOutKernel(KernelExec * kernel)313 void AddOutKernel(KernelExec *kernel) {
314 if (!lite::IsContain(this->out_kernels_, kernel)) {
315 this->out_kernels_.emplace_back(kernel);
316 }
317 }
318
FindAllInTensorIndex(const lite::Tensor * tensor)319 std::vector<size_t> FindAllInTensorIndex(const lite::Tensor *tensor) {
320 std::vector<size_t> indexes;
321 for (size_t i = 0; i < in_tensors().size(); i++) {
322 if (tensor == in_tensors().at(i)) {
323 indexes.emplace_back(i);
324 }
325 }
326 return indexes;
327 }
328
FindOutTensorIndex(const lite::Tensor * tensor)329 size_t FindOutTensorIndex(const lite::Tensor *tensor) {
330 size_t index = 0;
331 for (size_t i = 0; i < out_tensors().size(); i++) {
332 if (tensor == out_tensors().at(i)) {
333 index = i;
334 break;
335 }
336 }
337 return index;
338 }
339
RemoveInKernel(KernelExec * kernel)340 void RemoveInKernel(KernelExec *kernel) { (void)lite::VectorErase(&(this->in_kernels_), kernel); }
341
RemoveOutKernel(KernelExec * kernel)342 void RemoveOutKernel(KernelExec *kernel) { (void)lite::VectorErase(&(this->out_kernels_), kernel); }
343
set_in_kernels(const std::vector<KernelExec * > & kernel)344 void set_in_kernels(const std::vector<KernelExec *> &kernel) { this->in_kernels_ = kernel; }
345
set_out_kernels(const std::vector<KernelExec * > & kernel)346 void set_out_kernels(const std::vector<KernelExec *> &kernel) { this->out_kernels_ = kernel; }
347
in_kernels()348 const std::vector<KernelExec *> &in_kernels() const { return this->in_kernels_; }
349
out_kernels()350 const std::vector<KernelExec *> &out_kernels() const { return this->out_kernels_; }
351
352 virtual bool IsReady(const std::vector<lite::Tensor *> &in_tensor);
353
354 virtual void InitOutTensorInitRefCount(const std::vector<KernelExec *> *mask_kernels = nullptr);
355
desc()356 KernelKey desc() const { return desc_; }
357
set_desc(const KernelKey & kernel_key)358 void set_desc(const KernelKey &kernel_key) { desc_ = kernel_key; }
359
subgraph_type()360 SubGraphType subgraph_type() const { return this->subgraph_type_; }
361
set_context(const lite::InnerContext * context)362 void set_context(const lite::InnerContext *context) { context_ = context; }
363
Context()364 const lite::InnerContext *Context() const { return context_; }
365
366 virtual std::string ToString() const;
367
kernel()368 MSKernel *kernel() const { return kernel_.get(); }
369
SetOpenGLTextureEnable(bool enable)370 void SetOpenGLTextureEnable(bool enable) { enable_gl_texture_ = enable; }
371
GetOpenGLTextureEnable()372 bool GetOpenGLTextureEnable() const { return enable_gl_texture_; }
373
374 protected:
get_schema_primitive_kernel()375 Kernel *get_schema_primitive_kernel() const { return reinterpret_cast<Kernel *>(kernel_.get()); }
376
377 std::shared_ptr<MSKernel> kernel_ = nullptr;
378 KernelKey desc_;
379 // tensor will free in ~lite_session()
380 std::vector<KernelExec *> in_kernels_;
381 std::vector<KernelExec *> out_kernels_;
382 mutable std::vector<lite::Tensor *> mutable_in_tensors_;
383 mutable std::vector<lite::Tensor *> mutable_out_tensors_;
384 bool is_model_output_ = false;
385 SubGraphType subgraph_type_ = kNotSubGraph;
386 const lite::InnerContext *context_ = nullptr;
387 bool enable_gl_texture_ = false;
388 };
389
390 typedef LiteKernel *(*KernelCreator)(const std::vector<lite::Tensor *> &inputs,
391 const std::vector<lite::Tensor *> &outputs, OpParameter *parameter,
392 const lite::InnerContext *ctx, const KernelKey &desc);
393
394 template <class T>
LiteKernelCreator(const std::vector<lite::Tensor * > & inputs,const std::vector<lite::Tensor * > & outputs,OpParameter * parameter,const lite::InnerContext * ctx,const kernel::KernelKey & desc)395 LiteKernel *LiteKernelCreator(const std::vector<lite::Tensor *> &inputs, const std::vector<lite::Tensor *> &outputs,
396 OpParameter *parameter, const lite::InnerContext *ctx, const kernel::KernelKey &desc) {
397 if (parameter == nullptr) {
398 MS_LOG(ERROR) << "parameter is nullptr.";
399 return nullptr;
400 }
401 if (desc.data_type == kTypeUnknown) {
402 MS_LOG(WARNING) << "desc data_type is unknown.";
403 }
404 auto *kernel = new (std::nothrow) T(parameter, inputs, outputs, ctx);
405 if (kernel == nullptr) {
406 MS_LOG(ERROR) << "kernel: " << parameter->name_ << "is nullptr.";
407 free(parameter);
408 return nullptr;
409 }
410 return kernel;
411 }
412 } // namespace mindspore::kernel
413
414 #endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_EXEC_H_
415