1 /**
2 * Copyright 2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/litert/delegate/nnapi/op/conv_nnapi.h"
18 #include <algorithm>
19 #include <vector>
20 #include "src/litert/delegate/nnapi/nnapi_utils.h"
21 #include "src/common/utils.h"
22 #include "nnacl/op_base.h"
23 #include "nnacl/fp32/transpose_fp32.h"
24 #include "nnacl/transpose_parameter.h"
25
26 namespace mindspore {
27 namespace lite {
IsSupport()28 bool NNAPIConv::IsSupport() { return true; }
29
InitParams()30 int NNAPIConv::InitParams() {
31 auto conv = op_primitive_->value_as_Conv2DFusion();
32 MS_ASSERT(conv != nullptr);
33 group_ = static_cast<int>(conv->group());
34 in_channel_ = static_cast<int>(conv->in_channel());
35 out_channel_ = static_cast<int>(conv->out_channel());
36 is_dw_conv_ = group_ != 1 && group_ == in_channel_ && group_ == out_channel_;
37 is_group_conv_ = group_ != 1 && !is_dw_conv_;
38
39 pad_mode_ = static_cast<int>(conv->pad_mode());
40 if (conv->pad_list() != nullptr && conv->pad_list()->size() == DIMENSION_4D) {
41 pad_list_.push_back(static_cast<int>(*(conv->pad_list()->begin() + PAD_LEFT)));
42 pad_list_.push_back(static_cast<int>(*(conv->pad_list()->begin() + PAD_RIGHT)));
43 pad_list_.push_back(static_cast<int>(*(conv->pad_list()->begin() + PAD_UP)));
44 pad_list_.push_back(static_cast<int>(*(conv->pad_list()->begin() + PAD_DOWN)));
45 }
46 MS_CHECK_TRUE_RET(conv->stride() != nullptr && conv->stride()->size() == DIMENSION_2D, RET_ERROR);
47 strides_.push_back(static_cast<int>(*(conv->stride()->begin() + 1)));
48 strides_.push_back(static_cast<int>(*(conv->stride()->begin())));
49
50 MS_CHECK_TRUE_RET(conv->dilation() != nullptr && conv->dilation()->size() == DIMENSION_2D, RET_ERROR);
51 dilations_.push_back(static_cast<int>(*(conv->dilation()->begin() + 1)));
52 dilations_.push_back(static_cast<int>(*(conv->dilation()->begin())));
53 act_type_ = conv->activation_type();
54 return RET_OK;
55 }
56
AddAttributesForConv(ANeuralNetworksModel * nnapi_model,std::vector<mindspore::MSTensor> * all_tensors)57 int NNAPIConv::AddAttributesForConv(ANeuralNetworksModel *nnapi_model, std::vector<mindspore::MSTensor> *all_tensors) {
58 for (auto pad : pad_list_) {
59 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "pad", DataType::kNumberTypeInt32, pad) != RET_OK) {
60 MS_LOG(ERROR) << "Add paddings for conv to NNAPI model failed.";
61 return RET_ERROR;
62 }
63 }
64 if (pad_list_.empty()) {
65 // Use the implicit pad mode for NNAPI model.
66 MS_CHECK_TRUE_RET(pad_mode_ != PadType::Pad_pad, RET_ERROR);
67 auto pad_mode = pad_mode_ - 1; // the enum pad scheme of NNAPI is PAD_SAME and PAD_VALID.
68 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "pad_mode", DataType::kNumberTypeInt32, pad_mode) !=
69 RET_OK) {
70 MS_LOG(ERROR) << "Add pad mode for conv to NNAPI model failed.";
71 return RET_ERROR;
72 }
73 }
74
75 // set strides to NNAPI model.
76 for (auto stride : strides_) {
77 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "stride", DataType::kNumberTypeInt32, stride) != RET_OK) {
78 MS_LOG(ERROR) << "Add strides for conv to NNAPI model failed.";
79 return RET_ERROR;
80 }
81 }
82 // set group for grouped conv or multiplier for depthwise conv.
83 if (group_ != 1) {
84 int value = is_dw_conv_ ? 1 : group_;
85 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "group", DataType::kNumberTypeInt32, value) != RET_OK) {
86 MS_LOG(ERROR) << "Add activation type for conv to NNAPI model failed.";
87 return RET_ERROR;
88 }
89 }
90 // convert act_type to an input of nnapi node.
91 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "act_type", DataType::kNumberTypeInt32, act_type_) !=
92 RET_OK) {
93 MS_LOG(ERROR) << "Add activation type for conv to NNAPI model failed.";
94 return RET_ERROR;
95 }
96 // set nchw to an input of nnapi node.
97 if (AddScalarToNNAPIModel<bool>(nnapi_model, all_tensors, "nchw", DataType::kNumberTypeBool, false) != RET_OK) {
98 MS_LOG(ERROR) << "set nchw format for conv to NNAPI model failed.";
99 return RET_ERROR;
100 }
101 // grouped conv has no dilations.
102 if (!is_group_conv_) {
103 for (auto dilation : dilations_) {
104 if (AddScalarToNNAPIModel<int>(nnapi_model, all_tensors, "dilation", DataType::kNumberTypeInt32, dilation) !=
105 RET_OK) {
106 MS_LOG(ERROR) << "Add activation type for conv to NNAPI model failed.";
107 return RET_ERROR;
108 }
109 }
110 }
111 return RET_OK;
112 }
113
AddOpToNNAPIModel(ANeuralNetworksModel * nnapi_model,std::vector<mindspore::MSTensor> * all_tensors)114 int NNAPIConv::AddOpToNNAPIModel(ANeuralNetworksModel *nnapi_model, std::vector<mindspore::MSTensor> *all_tensors) {
115 MS_ASSERT(nnapi_model != nullptr && all_tensors != nullptr);
116 if (is_dw_conv_ && TransConvWeightFromKHWCToCHWK(nnapi_model, all_tensors) != RET_OK) {
117 MS_LOG(ERROR) << "Adjust weight for depthwise conv failed.";
118 return RET_ERROR;
119 }
120 if (InitNNAPIOpInOut(*all_tensors) != RET_OK) {
121 MS_LOG(ERROR) << "InitNNAPINodeInfo failed.";
122 return RET_ERROR;
123 }
124 if (in_tensors_.size() == kInputSize1) {
125 // has no bias, new a bias tensor with zero value.
126 auto weight_type = in_tensors_.at(1).DataType();
127 auto bias_type = (weight_type != DataType::kNumberTypeInt8 && weight_type != DataType::kNumberTypeUInt8)
128 ? weight_type
129 : DataType::kNumberTypeInt32;
130 MSTensorInfo bias_info{op_name_ + "_bias", bias_type, {out_channel_}, nullptr, 0};
131 if (AddTensorToNNAPIModel(nnapi_model, all_tensors, bias_info) != RET_OK) {
132 MS_LOG(ERROR) << "NNAPI does not support convolution without bias, and create zero-value tenosr failed.";
133 return RET_ERROR;
134 }
135 }
136
137 if (AddAttributesForConv(nnapi_model, all_tensors) != RET_OK) {
138 MS_LOG(ERROR) << "Add attributes for convolution failed.";
139 return RET_ERROR;
140 }
141
142 OperationCode node_type = is_dw_conv_ ? ANEURALNETWORKS_DEPTHWISE_CONV_2D
143 : (is_group_conv_ ? ANEURALNETWORKS_GROUPED_CONV_2D : ANEURALNETWORKS_CONV_2D);
144 if (nnapi_->ANeuralNetworksModel_addOperation(nnapi_model, node_type, input_indices_.size(), input_indices_.data(),
145 output_indices_.size(),
146 output_indices_.data()) != ANEURALNETWORKS_NO_ERROR) {
147 MS_LOG(ERROR) << "Add operation to NNAPI model failed.";
148 return RET_ERROR;
149 }
150 return RET_OK;
151 }
152
TransConvWeightFromKHWCToCHWK(ANeuralNetworksModel * nnapi_model,std::vector<mindspore::MSTensor> * all_tensors)153 int NNAPIConv::TransConvWeightFromKHWCToCHWK(ANeuralNetworksModel *nnapi_model,
154 std::vector<mindspore::MSTensor> *all_tensors) {
155 auto weight = inputs().at(1);
156 if (!weight.IsConst()) {
157 MS_LOG(ERROR) << "The weight of conv must be constant.";
158 return RET_ERROR;
159 }
160 auto shape = weight.Shape();
161 auto new_weight = weight.Clone();
162 MS_CHECK_TRUE_RET(new_weight != nullptr, RET_ERROR);
163 new_weight->SetQuantParams(weight.QuantParams());
164 // transpose weight from KHWC to CHWK.
165 std::vector<int64_t> new_shape = {shape.at(kNHWC_C), shape.at(kNHWC_H), shape.at(kNHWC_W), shape.at(kNHWC_N)};
166 new_weight->SetShape(new_shape);
167
168 TransposeParameter param;
169 param.strides_[DIMENSION_4D - 1] = 1;
170 param.out_strides_[DIMENSION_4D - 1] = 1;
171 for (int i = DIMENSION_4D - 2; i >= 0; i--) {
172 param.strides_[i] = shape[i + 1] * param.strides_[i + 1];
173 param.out_strides_[i] = new_shape[i + 1] * param.out_strides_[i + 1];
174 }
175 param.num_axes_ = DIMENSION_4D;
176 param.data_num_ = weight.ElementNum();
177 std::vector<int> perm = {3, 1, 2, 0};
178 memcpy(param.perm_, perm.data(), sizeof(int) * DIMENSION_4D);
179 std::vector<int> out_shape;
180 (void)std::transform(new_shape.begin(), new_shape.end(), std::back_inserter(out_shape),
181 [](int64_t x) { return static_cast<int>(x); });
182 int ret = 0;
183 if (weight.DataType() == DataType::kNumberTypeFloat32) {
184 ret = DoTransposeFp32(reinterpret_cast<float *>(weight.MutableData()),
185 reinterpret_cast<float *>(new_weight->MutableData()),
186 reinterpret_cast<const int *>(out_shape.data()), param.perm_, param.strides_,
187 param.out_strides_, param.data_num_, param.num_axes_);
188 } else {
189 MS_LOG(ERROR) << "Unsupported to pack depthwise conv weight";
190 delete new_weight;
191 return RET_ERROR;
192 }
193 if (ret != RET_OK) {
194 MS_LOG(ERROR) << "Transpose conv weight from KHWC to CHWK failed.";
195 delete new_weight;
196 return RET_ERROR;
197 }
198 if (AddNNAPIOperand(nnapi_model, *new_weight, static_cast<int>(all_tensors->size()), kNHWC_C) != RET_OK) {
199 MS_LOG(ERROR) << "Add depthwise conv weight to NNAPI model failed: " << op_name_;
200 delete new_weight;
201 return RET_ERROR;
202 }
203 in_tensors_.at(1) = *new_weight;
204 all_tensors->push_back(*new_weight);
205 op_attribute_tensors_.push_back(new_weight);
206 return RET_OK;
207 }
208 } // namespace lite
209 } // namespace mindspore
210