1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef MINDSPORE_LITE_SRC_LITE_KERNEL_H_
18 #define MINDSPORE_LITE_SRC_LITE_KERNEL_H_
19 #include <string>
20 #include <vector>
21 #include <memory>
22 #include <utility>
23 #include <algorithm>
24 #include "src/common/utils.h"
25 #include "src/common/log_util.h"
26 #ifdef ENABLE_ARM
27 #include <arm_neon.h>
28 #endif
29 #include "nnacl/op_base.h"
30 #include "src/inner_context.h"
31 #include "src/tensor.h"
32 #include "include/errorcode.h"
33 #include "schema/model_generated.h"
34 #include "include/context.h"
35 #include "include/api/kernel.h"
36 #include "src/cxx_api/tensor/tensor_impl.h"
37 #include "src/inner_kernel.h"
38 #ifndef DELEGATE_CLIP
39 #include "include/api/delegate.h"
40 #endif
41
42 namespace mindspore::kernel {
43 enum KERNEL_ARCH { kCPU, kGPU, kAPU, kNPU, kCustom, kDelegate, kKernelArch_MIN = kCPU, kKernelArch_MAX = kAPU };
44 static const char *const kBuiltin = "Builtin";
45
46 struct KernelKey {
47 KERNEL_ARCH arch = kCPU;
48 TypeId data_type = kTypeUnknown;
49 int type = 0;
50 std::string kernel_arch;
51 std::string provider{kBuiltin};
52
53 bool operator<(const KernelKey &dst) const {
54 if (provider != dst.provider) {
55 return provider < dst.provider;
56 } else if (kernel_arch != dst.kernel_arch) {
57 return kernel_arch < dst.kernel_arch;
58 } else if (arch != dst.arch) {
59 return arch < dst.arch;
60 } else if (data_type != dst.data_type) {
61 return data_type < dst.data_type;
62 } else {
63 return type < dst.type;
64 }
65 }
66 };
67
68 enum SubGraphType {
69 kNotSubGraph = 0,
70 kCpuFP32SubGraph,
71 kCpuFP16SubGraph,
72 kGpuFp32SubGraph,
73 kGpuFp16SubGraph,
74 kNpuSubGraph,
75 kApuSubGraph,
76 kCustomSubGraph
77 };
78
79 class LiteKernel {
80 public:
LiteKernel()81 LiteKernel() {
82 this->in_kernels_.clear();
83 this->out_kernels_.clear();
84 }
85
LiteKernel(std::shared_ptr<Kernel> kernel)86 explicit LiteKernel(std::shared_ptr<Kernel> kernel) : kernel_(kernel) {
87 this->in_kernels_.clear();
88 this->out_kernels_.clear();
89 }
90
91 virtual ~LiteKernel() = default;
92
Execute()93 virtual int Execute() { return DoExecute(); }
94
Execute(const KernelCallBack & before,const KernelCallBack & after)95 virtual int Execute(const KernelCallBack &before, const KernelCallBack &after) {
96 if (before != nullptr) {
97 if (!before(TensorVectorCast(this->in_tensors()), TensorVectorCast(this->out_tensors()),
98 {this->name(), schema::EnumNamePrimitiveType(this->type())})) {
99 MS_LOG(WARNING) << "run kernel before_callback failed, name: " << this->name();
100 }
101 }
102
103 auto ret = DoExecute();
104
105 if (after != nullptr) {
106 if (!after(TensorVectorCast(this->in_tensors()), TensorVectorCast(this->out_tensors()),
107 {this->name(), schema::EnumNamePrimitiveType(this->type())})) {
108 MS_LOG(WARNING) << "run kernel after_callback failed, name: " << this->name();
109 }
110 }
111 return ret;
112 }
113
114 // called while compiling graph
Prepare()115 virtual int Prepare() {
116 MS_ASSERT(kernel_ != nullptr);
117 return kernel_->Prepare();
118 }
119
Init()120 virtual int Init() {
121 MS_ASSERT(kernel_ != nullptr);
122 if (desc_.provider == kBuiltin) {
123 return std::static_pointer_cast<InnerKernel>(kernel_)->Init();
124 }
125 return mindspore::lite::RET_OK;
126 }
IsBuiltin()127 bool IsBuiltin() { return desc_.provider == kBuiltin; }
ReSize()128 virtual int ReSize() {
129 MS_ASSERT(kernel_ != nullptr);
130 return kernel_->ReSize();
131 }
132
op_parameter()133 OpParameter *op_parameter() const {
134 MS_ASSERT(kernel_ != nullptr);
135 if (desc_.provider == kBuiltin) {
136 return std::static_pointer_cast<InnerKernel>(kernel_)->op_parameter();
137 }
138 return nullptr;
139 }
140
name()141 std::string name() const {
142 MS_ASSERT(kernel_ != nullptr);
143 return kernel_->name();
144 }
145
set_name(const std::string & name)146 void set_name(const std::string &name) {
147 MS_ASSERT(kernel_ != nullptr);
148 kernel_->set_name(name);
149 }
150
Train()151 virtual int Train() {
152 MS_ASSERT(kernel_ != nullptr);
153 if (desc_.provider == kBuiltin) {
154 return std::static_pointer_cast<InnerKernel>(kernel_)->Train();
155 }
156 return mindspore::lite::RET_OK;
157 }
158
IsTrain()159 virtual bool IsTrain() const {
160 MS_ASSERT(kernel_ != nullptr);
161 if (desc_.provider == kBuiltin) {
162 return std::static_pointer_cast<InnerKernel>(kernel_)->IsTrain();
163 }
164 return false;
165 }
166
Eval()167 virtual int Eval() {
168 MS_ASSERT(kernel_ != nullptr);
169 if (desc_.provider == kBuiltin) {
170 return std::static_pointer_cast<InnerKernel>(kernel_)->Eval();
171 }
172 return mindspore::lite::RET_OK;
173 }
174
IsEval()175 virtual bool IsEval() const {
176 MS_ASSERT(kernel_ != nullptr);
177 if (desc_.provider == kBuiltin) {
178 return std::static_pointer_cast<InnerKernel>(kernel_)->IsEval();
179 }
180 return false;
181 }
182
183 virtual void SetTrainable(bool trainable = true) {
184 MS_ASSERT(kernel_ != nullptr);
185 if (desc_.provider == kBuiltin) {
186 std::static_pointer_cast<InnerKernel>(kernel_)->SetTrainable(trainable);
187 }
188 }
189
IsTrainable()190 virtual bool IsTrainable() const {
191 MS_ASSERT(kernel_ != nullptr);
192 if (desc_.provider == kBuiltin) {
193 return std::static_pointer_cast<InnerKernel>(kernel_)->IsTrainable();
194 }
195 return false;
196 }
197
198 int DoExecute();
199
set_is_model_output(bool is_model_output)200 void set_is_model_output(bool is_model_output) { this->is_model_output_ = is_model_output; }
201
is_model_output()202 bool is_model_output() const { return this->is_model_output_; }
203
InferShapeDone()204 bool InferShapeDone() const {
205 auto shape = out_tensors().front()->shape();
206 if (std::find(shape.begin(), shape.end(), -1) != shape.end()) {
207 return false;
208 }
209 return true;
210 }
211
type()212 schema::PrimitiveType type() const {
213 MS_ASSERT(kernel_ != nullptr);
214 return kernel_->type();
215 }
216
type_str()217 std::string type_str() const { return schema::EnumNamePrimitiveType(this->type()); }
218
set_in_tensors(const std::vector<lite::Tensor * > & in_tensors)219 void set_in_tensors(const std::vector<lite::Tensor *> &in_tensors) {
220 MS_ASSERT(kernel_ != nullptr);
221 if (desc_.provider == kBuiltin) {
222 std::static_pointer_cast<InnerKernel>(kernel_)->set_in_tensors(in_tensors);
223 } else {
224 std::vector<MSTensor> tensors_in;
225 std::transform(in_tensors.begin(), in_tensors.end(), std::back_inserter(tensors_in), [](lite::Tensor *tensor) {
226 auto impl = std::make_shared<mindspore::MSTensor::Impl>(tensor);
227 return mindspore::MSTensor(impl);
228 });
229 kernel_->set_inputs(tensors_in);
230 }
231 }
232
set_in_tensor(lite::Tensor * in_tensor,size_t index)233 void set_in_tensor(lite::Tensor *in_tensor, size_t index) {
234 MS_ASSERT(kernel_ != nullptr);
235 if (desc_.provider == kBuiltin) {
236 std::static_pointer_cast<InnerKernel>(kernel_)->set_in_tensor(in_tensor, index);
237 } else {
238 MS_ASSERT(index < kernel_->inputs().size());
239 auto impl = std::make_shared<mindspore::MSTensor::Impl>(in_tensor);
240 auto tensor_in = mindspore::MSTensor(impl);
241 kernel_->set_input(tensor_in, index);
242 }
243 }
244
set_out_tensors(const std::vector<lite::Tensor * > & out_tensors)245 void set_out_tensors(const std::vector<lite::Tensor *> &out_tensors) {
246 MS_ASSERT(kernel_ != nullptr);
247 if (desc_.provider == kBuiltin) {
248 std::static_pointer_cast<InnerKernel>(kernel_)->set_out_tensors(out_tensors);
249 } else {
250 std::vector<MSTensor> tensors_out;
251 std::transform(out_tensors.begin(), out_tensors.end(), std::back_inserter(tensors_out), [](lite::Tensor *tensor) {
252 auto impl = std::make_shared<mindspore::MSTensor::Impl>(tensor);
253 return mindspore::MSTensor(impl);
254 });
255 kernel_->set_outputs(tensors_out);
256 }
257 }
258
set_out_tensor(lite::Tensor * out_tensor,size_t index)259 virtual void set_out_tensor(lite::Tensor *out_tensor, size_t index) {
260 MS_ASSERT(kernel_ != nullptr);
261 if (desc_.provider == kBuiltin) {
262 std::static_pointer_cast<InnerKernel>(kernel_)->set_out_tensor(out_tensor, index);
263 } else {
264 MS_ASSERT(index < kernel_->outputs().size());
265 auto impl = std::make_shared<mindspore::MSTensor::Impl>(out_tensor);
266 auto tensor_out = mindspore::MSTensor(impl);
267 kernel_->set_output(tensor_out, index);
268 }
269 }
270
in_tensors()271 const std::vector<lite::Tensor *> &in_tensors() const {
272 MS_ASSERT(kernel_ != nullptr);
273 if (desc_.provider == kBuiltin) {
274 return std::static_pointer_cast<InnerKernel>(kernel_)->in_tensors();
275 } else {
276 auto &ms_tensors = kernel_->inputs();
277 mutable_in_tensors_.resize(ms_tensors.size());
278 (void)std::transform(
279 ms_tensors.begin(), ms_tensors.end(), mutable_in_tensors_.begin(),
280 [](const mindspore::MSTensor &tensor) { return static_cast<lite::Tensor *>(tensor.impl()->lite_tensor()); });
281
282 return mutable_in_tensors_;
283 }
284 }
285
out_tensors()286 const std::vector<lite::Tensor *> &out_tensors() const {
287 MS_ASSERT(kernel_ != nullptr);
288 if (desc_.provider == kBuiltin) {
289 return std::static_pointer_cast<InnerKernel>(kernel_)->out_tensors();
290 } else {
291 auto &ms_tensors = kernel_->outputs();
292 mutable_out_tensors_.resize(ms_tensors.size());
293 (void)std::transform(
294 ms_tensors.begin(), ms_tensors.end(), mutable_out_tensors_.begin(),
295 [](const mindspore::MSTensor &tensor) { return static_cast<lite::Tensor *>(tensor.impl()->lite_tensor()); });
296 return mutable_out_tensors_;
297 }
298 }
299
AddInKernel(LiteKernel * kernel)300 void AddInKernel(LiteKernel *kernel) {
301 if (!lite::IsContain(this->in_kernels_, kernel)) {
302 this->in_kernels_.emplace_back(kernel);
303 }
304 }
305
AddOutKernel(LiteKernel * kernel)306 void AddOutKernel(LiteKernel *kernel) {
307 if (!lite::IsContain(this->out_kernels_, kernel)) {
308 this->out_kernels_.emplace_back(kernel);
309 }
310 }
311
set_in_kernels(const std::vector<LiteKernel * > & kernel)312 void set_in_kernels(const std::vector<LiteKernel *> &kernel) { this->in_kernels_ = kernel; }
313
set_out_kernels(const std::vector<LiteKernel * > & kernel)314 void set_out_kernels(const std::vector<LiteKernel *> &kernel) { this->out_kernels_ = kernel; }
315
in_kernels()316 const std::vector<LiteKernel *> &in_kernels() const { return this->in_kernels_; }
317
out_kernels()318 const std::vector<LiteKernel *> &out_kernels() const { return this->out_kernels_; }
319
320 virtual bool IsReady(const std::vector<lite::Tensor *> &in_tensor);
321
322 virtual void InitOutTensorInitRefCount(const std::vector<LiteKernel *> *mask_kernels = nullptr);
323
desc()324 KernelKey desc() const { return desc_; }
325
set_desc(const KernelKey kernel_key)326 void set_desc(const KernelKey kernel_key) { desc_ = kernel_key; }
327
subgraph_type()328 SubGraphType subgraph_type() const { return this->subgraph_type_; }
329
set_context(const lite::InnerContext * context)330 void set_context(const lite::InnerContext *context) { context_ = context; }
331
Context()332 const lite::InnerContext *Context() const { return context_; }
333
334 virtual std::string ToString() const;
335
kernel()336 Kernel *kernel() { return kernel_.get(); }
337
338 protected:
339 std::shared_ptr<Kernel> kernel_ = nullptr;
340 KernelKey desc_;
341 // tensor will free in ~lite_session()
342 std::vector<LiteKernel *> in_kernels_;
343 std::vector<LiteKernel *> out_kernels_;
344 mutable std::vector<lite::Tensor *> mutable_in_tensors_;
345 mutable std::vector<lite::Tensor *> mutable_out_tensors_;
346 bool is_model_output_ = false;
347 SubGraphType subgraph_type_ = kNotSubGraph;
348 const lite::InnerContext *context_ = nullptr;
349 };
350
351 typedef InnerKernel *(*KernelCreator)(const std::vector<lite::Tensor *> &inputs,
352 const std::vector<lite::Tensor *> &outputs, OpParameter *parameter,
353 const lite::Context *ctx, const KernelKey &desc);
354
355 template <class T>
LiteKernelCreator(const std::vector<lite::Tensor * > & inputs,const std::vector<lite::Tensor * > & outputs,OpParameter * parameter,const lite::Context * ctx,const kernel::KernelKey & desc)356 kernel::InnerKernel *LiteKernelCreator(const std::vector<lite::Tensor *> &inputs,
357 const std::vector<lite::Tensor *> &outputs, OpParameter *parameter,
358 const lite::Context *ctx, const kernel::KernelKey &desc) {
359 if (parameter == nullptr) {
360 MS_LOG(ERROR) << "parameter is nullptr.";
361 return nullptr;
362 }
363 auto *kernel = new (std::nothrow) T(parameter, inputs, outputs, static_cast<const lite::InnerContext *>(ctx));
364 if (kernel == nullptr) {
365 MS_LOG(ERROR) << "kernel: " << parameter->name_ << "is nullptr.";
366 free(parameter);
367 return nullptr;
368 }
369 return kernel;
370 }
371 } // namespace mindspore::kernel
372
373 #endif // MINDSPORE_LITE_SRC_LITE_KERNEL_H_
374