1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "tools/converter/preprocess/image_preprocess.h"
18 #include "src/common/log_adapter.h"
19 #include "src/common/file_utils.h"
20 #include "include/errorcode.h"
21 #include "tools/converter/preprocess/opencv_utils.h"
22 namespace mindspore {
23 namespace lite {
24 namespace preprocess {
25 #ifdef MSLITE_DEPS_OPENCV
ReadImage(const std::string & image_path,cv::Mat * image)26 int ReadImage(const std::string &image_path, cv::Mat *image) {
27 if (image == nullptr) {
28 MS_LOG(ERROR) << "image is nullptr.";
29 return RET_ERROR;
30 }
31 *image = cv::imread(image_path);
32 if (image->empty() || image->data == nullptr) {
33 MS_LOG(ERROR) << "missing file, improper permissions, unsupported or invalid format.";
34 return RET_ERROR;
35 }
36 return RET_OK;
37 }
38
DecodeBuffer(const unsigned char * buffer,int length,cv::Mat * image)39 int DecodeBuffer(const unsigned char *buffer, int length, cv::Mat *image) {
40 if (image == nullptr) {
41 MS_LOG(ERROR) << "image is nullptr.";
42 return RET_ERROR;
43 }
44 *image = cv::imdecode(std::vector<uchar>(buffer, buffer + length), cv::IMREAD_COLOR);
45 if (image->empty() || image->data == nullptr) {
46 MS_LOG(ERROR) << "missing file, improper permissions, unsupported or invalid format.";
47 return RET_ERROR;
48 }
49 return RET_OK;
50 }
51
ConvertImageFormat(cv::Mat * image,cv::ColorConversionCodes to_format)52 int ConvertImageFormat(cv::Mat *image, cv::ColorConversionCodes to_format) {
53 if (image == nullptr) {
54 MS_LOG(ERROR) << "image is nullptr.";
55 return RET_ERROR;
56 }
57 if (to_format == cv::COLOR_COLORCVT_MAX) {
58 MS_LOG(ERROR) << "to_format is cv::COLOR_COLORCVT_MAX";
59 return RET_ERROR;
60 }
61 cv::cvtColor(*image, *image, to_format);
62 if (to_format == cv::COLOR_BGR2GRAY) {
63 image->convertTo(*image, CV_32FC1);
64 }
65 return RET_OK;
66 }
67
Normalize(cv::Mat * image,const std::vector<double> & mean,const std::vector<double> & standard_deviation)68 int Normalize(cv::Mat *image, const std::vector<double> &mean, const std::vector<double> &standard_deviation) {
69 if (image == nullptr) {
70 MS_LOG(ERROR) << "image is nullptr.";
71 return RET_ERROR;
72 }
73 if (static_cast<int>(mean.size()) != image->channels() ||
74 static_cast<int>(standard_deviation.size()) != image->channels()) {
75 MS_LOG(ERROR) << "mean size:" << mean.size() << " != image->channels:" << image->channels()
76 << " or scale size:" << standard_deviation.size() << " !=image->dims:" << image->channels();
77 return RET_ERROR;
78 }
79 std::vector<cv::Mat> channels(image->channels());
80 cv::split(*image, channels);
81 for (size_t i = 0; i < channels.size(); i++) {
82 MS_ASSERT(standard_deviation[i] > 0);
83 channels[i].convertTo(channels[i], CV_32FC1, 1.0 / standard_deviation[i], (0.0 - mean[i]) / standard_deviation[i]);
84 }
85 cv::merge(channels, *image);
86 return RET_OK;
87 }
88
Resize(cv::Mat * image,int width,int height,cv::InterpolationFlags resize_method)89 int Resize(cv::Mat *image, int width, int height, cv::InterpolationFlags resize_method) {
90 if (image == nullptr) {
91 MS_LOG(ERROR) << "image is nullptr.";
92 return RET_ERROR;
93 }
94 if (width <= 0 || height <= 0) {
95 MS_LOG(ERROR) << "Both width and height must be > 0."
96 << " width:" << width << " height:" << height;
97 return RET_ERROR;
98 }
99 if (resize_method == cv::INTER_MAX) {
100 MS_LOG(ERROR) << "resize method is cv::INTER_MAX";
101 return RET_ERROR;
102 }
103 cv::resize(*image, *image, cv::Size(width, height), 0, 0, resize_method);
104 return RET_OK;
105 }
106
CenterCrop(cv::Mat * image,int width,int height)107 int CenterCrop(cv::Mat *image, int width, int height) {
108 if (image == nullptr) {
109 MS_LOG(ERROR) << "image is nullptr.";
110 return RET_ERROR;
111 }
112 if (width <= 0 || height <= 0) {
113 MS_LOG(ERROR) << "Both width and height must be > 0."
114 << " width:" << width << " height:" << height;
115 return RET_ERROR;
116 }
117 if (width > image->cols || height > image->rows) {
118 MS_LOG(ERROR) << "width:" << width << " > "
119 << "image->cols:" << image->cols << " or"
120 << "height:" << height << " > "
121 << "image->rows:" << image->rows;
122 return RET_ERROR;
123 }
124 const int offsetW = (image->cols - width) / 2;
125 const int offsetH = (image->rows - height) / 2;
126 const cv::Rect roi(offsetW, offsetH, width, height);
127 cv::Mat image_object = *(image);
128 *(image) = image_object(roi).clone();
129 return RET_OK;
130 }
131
ImagePreProcess(const ImagePreProcessParam & image_preprocess_param,cv::Mat * image,void ** data,size_t * size)132 int ImagePreProcess(const ImagePreProcessParam &image_preprocess_param, cv::Mat *image, void **data, size_t *size) {
133 if (image == nullptr || data == nullptr || size == nullptr) {
134 MS_LOG(ERROR) << "data or size is nullptr.";
135 return RET_NULL_PTR;
136 }
137 int ret;
138 if (image_preprocess_param.image_to_format_code != cv::COLOR_COLORCVT_MAX) {
139 ret = ConvertImageFormat(image, image_preprocess_param.image_to_format_code);
140 if (ret != RET_OK) {
141 MS_LOG(ERROR) << "convert image format failed.";
142 return ret;
143 }
144 }
145 // normalize_mean and normalize_std vector size must equal.
146 if (!image_preprocess_param.normalize_mean.empty() || !image_preprocess_param.normalize_std.empty()) {
147 ret = Normalize(image, image_preprocess_param.normalize_mean, image_preprocess_param.normalize_std);
148 if (ret != RET_OK) {
149 MS_LOG(ERROR) << "image normalize failed.";
150 return ret;
151 }
152 }
153
154 if (image_preprocess_param.resize_method != cv::INTER_MAX) {
155 ret = Resize(image, image_preprocess_param.resize_width, image_preprocess_param.resize_height,
156 image_preprocess_param.resize_method);
157 if (ret != RET_OK) {
158 MS_LOG(ERROR) << "image reize failed.";
159 return ret;
160 }
161 }
162
163 if (image_preprocess_param.center_crop_height != -1 && image_preprocess_param.center_crop_width != -1) {
164 ret = CenterCrop(image, image_preprocess_param.center_crop_width, image_preprocess_param.center_crop_height);
165 if (ret != RET_OK) {
166 MS_LOG(ERROR) << "image center crop failed.";
167 return ret;
168 }
169 }
170
171 ret = GetMatData(*image, data, size);
172 if (ret != RET_OK) {
173 MS_LOG(ERROR) << "Get mat data failed.";
174 return ret;
175 }
176 return RET_OK;
177 }
178 #endif
179
PreProcess(const preprocess::DataPreProcessParam & data_pre_process_param,const std::string & input_name,size_t image_index,mindspore::MSTensor * tensor)180 int PreProcess(const preprocess::DataPreProcessParam &data_pre_process_param, const std::string &input_name,
181 size_t image_index, mindspore::MSTensor *tensor) {
182 if (tensor == nullptr) {
183 MS_LOG(ERROR) << "tensor is nullptr.";
184 return RET_NULL_PTR;
185 }
186 size_t size;
187 char *data_buffer = nullptr;
188 auto ret =
189 PreProcess(data_pre_process_param, input_name, image_index, reinterpret_cast<void **>(&data_buffer), &size);
190 if (data_buffer == nullptr || size == 0) {
191 MS_LOG(ERROR) << "data_buffer is nullptr or size == 0";
192 return RET_ERROR;
193 }
194 if (ret != RET_OK) {
195 MS_LOG(ERROR) << "Preprocess failed.";
196 delete[] data_buffer;
197 return RET_ERROR;
198 }
199 auto data = tensor->MutableData();
200 if (data == nullptr) {
201 MS_LOG(ERROR) << "Get tensor MutableData return nullptr";
202 delete[] data_buffer;
203 return RET_NULL_PTR;
204 }
205 if (size != tensor->DataSize()) {
206 MS_LOG(ERROR) << "the input data is not consistent with model input, file_size: " << size
207 << " input tensor size: " << tensor->DataSize();
208 delete[] data_buffer;
209 return RET_ERROR;
210 }
211 if (memcpy_s(data, tensor->DataSize(), data_buffer, size) != EOK) {
212 MS_LOG(ERROR) << "memcpy data failed.";
213 delete[] data_buffer;
214 return RET_ERROR;
215 }
216 delete[] data_buffer;
217 return RET_OK;
218 }
219
220 // preprocess input batch
PreProcessBatch(const preprocess::DataPreProcessParam & data_pre_process_param,const std::string & input_name,lite::Tensor * tensor)221 int PreProcessBatch(const preprocess::DataPreProcessParam &data_pre_process_param, const std::string &input_name,
222 lite::Tensor *tensor) {
223 CHECK_NULL_RETURN(tensor);
224 for (int image_index = 0; image_index < data_pre_process_param.calibrate_size; image_index++) {
225 size_t size;
226 char *data_buffer = nullptr;
227 auto ret =
228 PreProcess(data_pre_process_param, input_name, image_index, reinterpret_cast<void **>(&data_buffer), &size);
229 if (data_buffer == nullptr || size == 0) {
230 MS_LOG(ERROR) << "data_buffer is nullptr or size == 0";
231 return RET_ERROR;
232 }
233 if (ret != RET_OK) {
234 MS_LOG(ERROR) << "Preprocess failed.";
235 delete[] data_buffer;
236 return RET_ERROR;
237 }
238 if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeFloat) {
239 auto data = reinterpret_cast<float *>(tensor->MutableData());
240 CHECK_NULL_RETURN(data);
241 if (memcpy_s(data + image_index * tensor->ElementsNum(), tensor->Size(), data_buffer, size) != EOK) {
242 MS_LOG(ERROR) << "memcpy data failed.";
243 delete[] data_buffer;
244 return RET_ERROR;
245 }
246 } else {
247 MS_LOG(ERROR) << "Not supported data_type: " << tensor->data_type();
248 return RET_ERROR;
249 }
250 delete[] data_buffer;
251 }
252 return RET_OK;
253 }
254
PreProcess(const DataPreProcessParam & data_pre_process_param,const std::string & input_name,size_t image_index,void ** data,size_t * size)255 int PreProcess(const DataPreProcessParam &data_pre_process_param, const std::string &input_name, size_t image_index,
256 void **data, size_t *size) {
257 if (data == nullptr || size == nullptr) {
258 MS_LOG(ERROR) << "data or size is nullptr.";
259 return RET_NULL_PTR;
260 }
261
262 if (data_pre_process_param.calibrate_path_vector.find(input_name) ==
263 data_pre_process_param.calibrate_path_vector.end()) {
264 MS_LOG(ERROR) << "Cant find input:" << input_name;
265 return RET_INPUT_PARAM_INVALID;
266 }
267 auto data_path = data_pre_process_param.calibrate_path_vector.at(input_name).at(image_index);
268 #ifndef MSLITE_DEPS_OPENCV
269 if (data_pre_process_param.input_type == BIN) {
270 #else
271 if (data_pre_process_param.input_type == IMAGE) {
272 cv::Mat mat;
273 auto ret = ReadImage(data_path, &mat);
274 if (ret != RET_OK) {
275 MS_LOG(ERROR) << "Read image failed.";
276 return ret;
277 }
278 ret = ImagePreProcess(data_pre_process_param.image_pre_process, &mat, data, size);
279 if (ret != RET_OK) {
280 MS_LOG(ERROR) << "Image Preprocess failed.";
281 return ret;
282 }
283 } else if (data_pre_process_param.input_type == BIN) {
284 #endif
285 *data = ReadFile(data_path.c_str(), size);
286 if (*data == nullptr || *size == 0) {
287 MS_LOG(ERROR) << "ReadFile return nullptr";
288 return RET_NULL_PTR;
289 }
290 } else {
291 MS_LOG(ERROR) << "INPUT ILLEGAL: input_type must be IMAGE|BIN.";
292 return RET_ERROR;
293 }
294 return RET_OK;
295 }
296 } // namespace preprocess
297 } // namespace lite
298 } // namespace mindspore
299