1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/delegate/tensorrt/op/deconvolution_tensorrt.h"
18 #include "src/delegate/tensorrt/op/activation_tensorrt.h"
19 #include "nnacl/pack.h"
20
21 namespace mindspore::lite {
IsSupport(const schema::Primitive * primitive,const std::vector<mindspore::MSTensor> & in_tensors,const std::vector<mindspore::MSTensor> & out_tensors)22 int DeconvolutionTensorRT::IsSupport(const schema::Primitive *primitive,
23 const std::vector<mindspore::MSTensor> &in_tensors,
24 const std::vector<mindspore::MSTensor> &out_tensors) {
25 if (!IsShapeKnown()) {
26 MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_;
27 return RET_ERROR;
28 }
29 if (in_tensors.size() != INPUT_SIZE2 && in_tensors.size() != INPUT_SIZE3) {
30 MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size();
31 return RET_ERROR;
32 }
33 if (out_tensors.size() != 1) {
34 MS_LOG(ERROR) << "Unsupported output tensor size, size is " << out_tensors.size();
35 return RET_ERROR;
36 }
37 if (in_tensors[0].format() != Format::NHWC && in_tensors[0].format() != Format::NCHW) {
38 MS_LOG(ERROR) << "Unsupported input tensor format of " << in_tensors[0].format();
39 return RET_ERROR;
40 }
41 return RET_OK;
42 }
AddInnerOp(nvinfer1::INetworkDefinition * network)43 int DeconvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) {
44 if (network == nullptr) {
45 MS_LOG(ERROR) << "network is invalid";
46 return RET_ERROR;
47 }
48 const schema::Conv2dTransposeFusion *deconv_op = this->op_primitive_->value_as_Conv2dTransposeFusion();
49 if (deconv_op == nullptr) {
50 MS_LOG(ERROR) << "op action convert failed";
51 return RET_ERROR;
52 }
53 nvinfer1::ITensor *deconv_input = tensorrt_in_tensors_[0].trt_tensor_;
54 if (tensorrt_in_tensors_[0].trt_tensor_->getDimensions().nbDims == DIMENSION_4D &&
55 tensorrt_in_tensors_[0].format_ == Format::NHWC) {
56 // transpose: NHWC->NCHW
57 nvinfer1::IShuffleLayer *transpose_layer_in = NHWC2NCHW(network, *tensorrt_in_tensors_[0].trt_tensor_);
58 if (transpose_layer_in == nullptr) {
59 MS_LOG(ERROR) << "transpose: NHWC->NCHW failed";
60 return RET_ERROR;
61 }
62 transpose_layer_in->setName((op_name_ + "_transpose2NCHW").c_str());
63 deconv_input = transpose_layer_in->getOutput(0);
64 }
65
66 // transpose weight
67 const mindspore::MSTensor &weight_tensor = in_tensors_[1];
68 nvinfer1::Weights kernelWeights = lite::TransposeWeight(weight_tensor, &pack_weight_);
69
70 // deconv basic params
71 int nbOutputMaps = weight_tensor.Shape()[0];
72 if (nbOutputMaps <= 0) {
73 MS_LOG(ERROR) << "out_channel is invalid";
74 return RET_ERROR;
75 }
76
77 auto kernel_size = deconv_op->kernel_size();
78 if (kernel_size == nullptr) {
79 MS_LOG(ERROR) << "kernel_size is null";
80 return RET_ERROR;
81 }
82 nvinfer1::Dims kernelSize = lite::ConvertCudaDims(std::vector<int64_t>(kernel_size->begin(), kernel_size->end()));
83
84 // bias
85 nvinfer1::Weights biasWeights{};
86 if (in_tensors_.size() >= INPUT_SIZE3) {
87 biasWeights = lite::ConvertWeight(in_tensors_[INPUT_SIZE3 - 1]);
88 } else {
89 biasWeights.type = ConvertDataType(weight_tensor.DataType());
90 biasWeights.count = 0;
91 biasWeights.values = nullptr;
92 }
93
94 nvinfer1::IDeconvolutionLayer *deconv_layer =
95 network->addDeconvolutionNd(*deconv_input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);
96
97 if (deconv_layer == nullptr) {
98 MS_LOG(ERROR) << "DeconvolutionLayer failed";
99 return RET_ERROR;
100 }
101 deconv_layer->setName((op_name_ + "_deconv").c_str());
102
103 // set extra params
104 SetAttributes(deconv_op, deconv_layer);
105
106 // add activation
107 nvinfer1::ILayer *activation_layer = nullptr;
108 if (deconv_op->activation_type() == schema::ActivationType::ActivationType_NO_ACTIVATION) {
109 activation_layer = deconv_layer;
110 } else {
111 activation_layer =
112 ActivationTensorRT::AddActivation(network, deconv_op->activation_type(), 0, deconv_layer->getOutput(0));
113 if (activation_layer == nullptr) {
114 MS_LOG(ERROR) << "addActivation for conv failed";
115 return RET_ERROR;
116 }
117 activation_layer->setName((op_name_ + "_activation").c_str());
118 }
119 activation_layer->getOutput(0)->setName((op_name_ + "_output").c_str());
120 this->AddInnerOutTensors(ITensorHelper{activation_layer->getOutput(0), Format::NCHW});
121 return RET_OK;
122 }
123
SetAttributes(const schema::Conv2dTransposeFusion * ms_op,nvinfer1::IDeconvolutionLayer * decon_layer)124 void DeconvolutionTensorRT::SetAttributes(const schema::Conv2dTransposeFusion *ms_op,
125 nvinfer1::IDeconvolutionLayer *decon_layer) {
126 // kernel_size
127 auto kernel_size = ms_op->kernel_size();
128 auto kernel_size_val = std::vector<int64_t>(kernel_size->begin(), kernel_size->end());
129 nvinfer1::Dims kernel_size_dims = lite::ConvertCudaDims(kernel_size_val);
130 decon_layer->setKernelSizeNd(kernel_size_dims);
131
132 // nbOutputMaps
133 int32_t nbOutputMaps = static_cast<int32_t>(ms_op->out_channel());
134 decon_layer->setNbOutputMaps(nbOutputMaps);
135
136 // stride
137 auto stride = ms_op->stride();
138 auto stride_val = std::vector<int64_t>(stride->begin(), stride->end());
139 nvinfer1::Dims stride_dims = lite::ConvertCudaDims(stride_val);
140 decon_layer->setStrideNd(stride_dims);
141
142 // nbGroups
143 int32_t nbGroups = static_cast<int32_t>(ms_op->group());
144 decon_layer->setNbGroups(nbGroups);
145
146 // padding
147 schema::PadMode pad_mode = ms_op->pad_mode();
148 if (pad_mode == schema::PadMode::PadMode_SAME) {
149 decon_layer->setPaddingMode(nvinfer1::PaddingMode::kSAME_UPPER);
150 } else {
151 auto padding = ms_op->pad_list();
152 auto out_pad = ms_op->output_paddings();
153 if (padding == nullptr || out_pad == nullptr) {
154 MS_LOG(WARNING) << "on pad value of " << op_name_;
155 return;
156 }
157 auto padding_val = std::vector<int64_t>(padding->begin(), padding->end());
158 auto out_pad_val = std::vector<int64_t>(out_pad->begin(), out_pad->end()); // h, w
159 if (out_pad_val.size() != DIMENSION_2D || padding_val.size() != DIMENSION_4D) {
160 MS_LOG(ERROR) << "invalid size of pad " << op_name_;
161 return;
162 }
163 nvinfer1::Dims dims_pre{};
164 dims_pre.nbDims = 2;
165 dims_pre.d[0] = padding_val[0]; // up
166 dims_pre.d[1] = padding_val[2]; // left
167 decon_layer->setPrePadding(dims_pre);
168 nvinfer1::Dims dims_post{};
169 dims_post.nbDims = 2;
170 dims_post.d[0] = padding_val[1] - out_pad_val[0]; // down
171 dims_post.d[1] = padding_val[3] - out_pad_val[1]; // right
172 decon_layer->setPostPadding(dims_post);
173 }
174 }
175
~DeconvolutionTensorRT()176 DeconvolutionTensorRT::~DeconvolutionTensorRT() {
177 if (pack_weight_ != nullptr) {
178 free(pack_weight_);
179 pack_weight_ = nullptr;
180 }
181 }
182 } // namespace mindspore::lite
183