1 /**
2 * Copyright 2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #ifndef MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_TENSORRT_UTILS_H_
17 #define MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_TENSORRT_UTILS_H_
18 #include <experimental/optional>
19 #include <vector>
20 #include <NvInfer.h>
21 #include <NvInferVersion.h>
22 #include <memory>
23 #include <string>
24 #include <map>
25 #include "src/extendrt/delegate/tensorrt/tensorrt_context.h"
26 #include "src/extendrt/delegate/tensorrt/tensor_info.h"
27 #include "src/extendrt/delegate/tensorrt/cuda_impl/cublas_utils.h"
28 #include "mindspore/core/ir/dtype/type_id.h"
29 #include "schema/ops_generated.h"
30 #include "nnacl/pack.h"
31 #include "include/api/context.h"
32 #include "mindapi/base/types.h"
33
34 #define kNCHW_N 0
35 #define kNCHW_C 1
36 #define kNCHW_H 2
37 #define kNCHW_W 3
38 #define kNHWC_N 0
39 #define kNHWC_H 1
40 #define kNHWC_W 2
41 #define kNHWC_C 3
42
43 namespace mindspore::lite {
44 #define TRT_VERSION_GE(major, minor) \
45 (NV_TENSORRT_MAJOR > major) || ((NV_TENSORRT_MAJOR == major && NV_TENSORRT_MINOR >= minor))
46 #define TRT_VERSION_LS(major, minor) \
47 (NV_TENSORRT_MAJOR < major) || ((NV_TENSORRT_MAJOR == major && NV_TENSORRT_MINOR < minor))
48 struct ActivationParams {
49 nvinfer1::ActivationType activation_type;
50 bool has_alpha;
51 float alpha;
52 bool has_beta;
53 float beta;
54 };
55
56 typedef union float32_bits {
57 unsigned int u;
58 float f;
59 } float32_bits;
60
61 #ifdef PROFILER_
62 struct SimpleProfiler : public nvinfer1::IProfiler {
63 struct Record {
64 float time{0};
65 int count{0};
66 };
67
68 virtual void reportLayerTime(const char *layerName, float ms) noexcept;
69
70 explicit SimpleProfiler(const char *name,
71 const std::vector<SimpleProfiler> &srcProfilers = std::vector<SimpleProfiler>());
72
73 friend std::ostream &operator<<(std::ostream &out, const SimpleProfiler &value);
74
75 private:
76 std::string mName_;
77 std::vector<std::string> mLayerNames_;
78 std::map<std::string, Record> mProfile_;
79 };
80 #endif
81
82 // Convert Tensor data to Cuda dims.
83 nvinfer1::Dims ConvertCudaDims(const std::vector<int> &data);
84
85 nvinfer1::Dims ConvertCudaDims(int data, size_t size);
86
87 nvinfer1::Dims ConvertCudaDims(const TensorInfo &ms_tensor);
88
89 std::string CudaDimsAsString(const nvinfer1::Dims &dims);
90
91 std::vector<int32_t> ConvertTensorAsIntVector(const TensorInfo &ms_tensor);
92
93 bool SameDims(nvinfer1::Dims dims, const std::vector<int64_t> &shape);
94
95 std::vector<int64_t> ConvertMSShape(const nvinfer1::Dims dims);
96
97 std::vector<int64_t> NHWC2NCHW(std::vector<int64_t> nhwc_shape);
98
99 nvinfer1::DataType ConvertDataType(DataType type_id);
100
101 cudaDataType ConvertDataType(nvinfer1::DataType type_id);
102
103 nvinfer1::IShuffleLayer *NHWC2NCHW(TensorRTContext *ctx, const nvinfer1::ITensor &input);
104
105 nvinfer1::IShuffleLayer *NCHW2NHWC(TensorRTContext *ctx, const nvinfer1::ITensor &input);
106
107 std::experimental::optional<ActivationParams> TryConvertActivationType(ActivationType activation_type);
108
109 nvinfer1::ITensor *ConvertConstantTensor(TensorRTContext *ctx, const TensorInfo &ms_tensor, const std::string &op_name);
110
111 nvinfer1::ITensor *ConvertTensorWithExpandDims(TensorRTContext *ctx, const TensorInfo &ms_tensor,
112 const std::vector<int64_t> &expect_shape, const std::string &op_name);
113
114 nvinfer1::ITensor *ConvertScalarToITensor(TensorRTContext *ctx, size_t shape_size, const void *value,
115 const DataType data_type, const std::string &op_name);
116
117 nvinfer1::ITensor *ConvertScalarToITensor(TensorRTContext *ctx, size_t shape_size, const TensorInfo &ms_tensor,
118 const DataType data_type, const std::string &op_name);
119
120 nvinfer1::ITensor *ConvertConstantTensorWithDims(TensorRTContext *ctx, const TensorInfo &ms_tensor,
121 const std::vector<int64_t> &expect_shape, const std::string &op_name);
122
123 nvinfer1::Weights TransposeWeight2D(const TensorInfo &ms_tensor, void **pack_weight);
124
125 nvinfer1::Weights ConvertWeight(const TensorInfo &ms_tensor);
126
127 nvinfer1::ITensor *TRTTensorCast(TensorRTContext *ctx, nvinfer1::ITensor *tensor, nvinfer1::DataType data_type,
128 const std::string &name);
129
130 int SetCudaDevice(std::shared_ptr<GPUDeviceInfo> device_info_);
131
132 int SetCudaDevice(int device_id);
133
134 Format GetOutputFormat(Format input_format, nvinfer1::Permutation perm);
135
136 int ConvertAxisFromNHWC2NCHW(int nhwc_axis);
137
138 void PackNHWCToNCHWFp16(const void *src, void *dst, size_t batch, size_t plane, size_t channel, size_t task_id,
139 size_t thread_count);
140
141 std::string GetTensorFormat(nvinfer1::ITensor *trt_tensor, mindspore::Format format, bool is_same);
142
143 std::string GetTensorFormat(ITensorHelper tensor_helper);
144
145 std::string GetTensorFormat(nvinfer1::ITensor *trt_tensors);
146
147 std::experimental::optional<nvinfer1::ReduceOperation> TryConvertTRTReduceMode(ReduceMode mode);
148
149 int PreprocessInputs2SameDim(TensorRTContext *ctx, ITensorHelper input_tensor_helper, ITensorHelper *out_tensor_helper);
150
151 int GetDimsVolume(const nvinfer1::Dims &dims);
152
153 int GetDimsVolume(const std::vector<int64_t> &shape);
154
155 std::experimental::optional<nvinfer1::Dims> SqueezeDims(const nvinfer1::Dims &in_dims, int pos);
156
157 std::experimental::optional<nvinfer1::Dims> UnsqueezeDims(const nvinfer1::Dims &in_dims, int pos, int val);
158
159 nvinfer1::ITensor *Reshape(TensorRTContext *ctx, nvinfer1::ITensor *input, const std::vector<int64_t> &shape);
160
161 nvinfer1::ITensor *Reshape(TensorRTContext *ctx, nvinfer1::ITensor *input, const nvinfer1::Dims &shape);
162
163 nvinfer1::ITensor *ConvertConstantTensor1D(TensorRTContext *ctx, int *weights_vec, nvinfer1::DataType data_type);
164
165 int ParseData2Vector(const TensorInfo &ms_tensor, std::vector<float> *dst);
166
167 void DebugDims(const std::string &key, const nvinfer1::Dims &dims);
168
169 nvinfer1::ITensor *ExpandDim(TensorRTContext *ctx, nvinfer1::ITensor *input_tensor, int axis);
170
171 nvinfer1::ITensor *Broadcast(TensorRTContext *ctx, nvinfer1::ITensor *input, nvinfer1::ITensor *shape);
172
173 template <typename T>
174 nvinfer1::DataType GetNvinferDataType();
175
176 template <typename T1, typename T2>
SameDims(const std::vector<T1> & shape1,const std::vector<T2> & shape2)177 bool SameDims(const std::vector<T1> &shape1, const std::vector<T2> &shape2) {
178 if (shape1.size() != shape2.size()) {
179 return false;
180 }
181 for (size_t i = 0; i < shape1.size(); i++) {
182 if (std::abs(shape1[i] - shape2[i]) > 1e-6) {
183 return false;
184 }
185 }
186 return true;
187 }
188
189 template <typename T>
ConvertCudaDims(const std::vector<T> & shape)190 nvinfer1::Dims ConvertCudaDims(const std::vector<T> &shape) {
191 nvinfer1::Dims dims{};
192 dims.nbDims = -1;
193 if (!shape.empty() && shape.size() <= static_cast<size_t>(dims.MAX_DIMS)) {
194 dims.nbDims = shape.size();
195 for (int i = 0; i < dims.nbDims; i++) {
196 dims.d[i] = static_cast<int>(shape[i]);
197 }
198 } else {
199 MS_LOG(INFO) << "ms shape is invalid!shape size: " << shape.size();
200 }
201 return dims;
202 }
203
IntToSize(int u)204 inline size_t IntToSize(int u) {
205 if (u < 0) {
206 MS_LOG(WARNING) << "The int value(" << u << ") is less than 0.";
207 return SIZE_MAX;
208 }
209 return static_cast<size_t>(u);
210 }
211 template <typename T>
Data2Vector(std::vector<float> * dst,const void * src)212 void Data2Vector(std::vector<float> *dst, const void *src) {
213 auto src_ptr = static_cast<const T *>(src);
214 for (size_t i = 0; i < dst->size(); i++) {
215 dst->at(i) = static_cast<float>(src_ptr[i]);
216 }
217 }
218 } // namespace mindspore::lite
219 #endif // MINDSPORE_LITE_SRC_EXTENDRT_DELEGATE_TENSORRT_TENSORRT_UTILS_H_
220