1 /**
2 * Copyright 2020 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "bboxop_common.h"
18
19 #include <memory>
20 #include <string>
21 #include <vector>
22 #include <iostream>
23
24 #include <stdio.h>
25
26 #include "./tinyxml2.h"
27 #include "opencv2/opencv.hpp"
28 #include "utils/ms_utils.h"
29 #include "minddata/dataset/core/cv_tensor.h"
30 #include "minddata/dataset/util/path.h"
31 #include "minddata/dataset/include/dataset/constants.h"
32 #include "utils/log_adapter.h"
33
34 using namespace mindspore::dataset;
35 using namespace UT::CVOP::BBOXOP;
36 using tinyxml2::XMLDocument;
37 using tinyxml2::XMLElement;
38 using tinyxml2::XMLError;
39
40 const char kAnnotationsFolder[] = "/Annotations/";
41 const char kImagesFolder[] = "/JPEGImages/";
42 const char kExpectedName[] = "apple_expect_";
43 const char kActualName[] = "Actual";
44 const char kAnnotExt[] = ".xml";
45 const char kImageExt[] = ".jpg";
46
BBoxOpCommon()47 BBoxOpCommon::BBoxOpCommon() {}
48
~BBoxOpCommon()49 BBoxOpCommon::~BBoxOpCommon() {}
50
SetUp()51 void BBoxOpCommon::SetUp() {
52 MS_LOG(INFO) << "starting test.";
53 image_folder_build_ = "data/dataset/imagefolder/";
54 image_folder_src_ = "../../../../../tests/ut/data/dataset/imagefolder/";
55 std::string dir_path = "data/dataset/testVOC2012_2";
56 GetInputImagesAndAnnotations(dir_path);
57 }
58
GetInputImagesAndAnnotations(const std::string & dir,std::size_t num_of_samples)59 void BBoxOpCommon::GetInputImagesAndAnnotations(const std::string &dir, std::size_t num_of_samples) {
60 std::string images_path = dir + std::string(kImagesFolder);
61 std::string annots_path = dir + std::string(kAnnotationsFolder);
62 Path dir_path(images_path);
63 std::shared_ptr<Path::DirIterator> image_dir_itr = Path::DirIterator::OpenDirectory(&dir_path);
64 std::vector<std::string> paths_to_fetch;
65 if (!dir_path.Exists()) {
66 MS_LOG(ERROR) << "Images folder was not found : " + images_path;
67 EXPECT_TRUE(dir_path.Exists());
68 }
69 // get image file paths
70 while (image_dir_itr->HasNext()) {
71 Path image_path = image_dir_itr->Next();
72 if (image_path.Extension() == std::string(kImageExt)) {
73 paths_to_fetch.push_back(image_path.ToString());
74 }
75 }
76 // sort fetched files
77 std::sort(paths_to_fetch.begin(), paths_to_fetch.end());
78 std::size_t files_fetched = 0;
79 for (const auto &image_file : paths_to_fetch) {
80 std::string image_ext = std::string(kImageExt);
81 std::string annot_file = image_file;
82 std::size_t pos = 0;
83 // first replace the Image dir with the Annotation dir.
84 if ((pos = image_file.find(std::string(kImagesFolder), 0)) != std::string::npos) {
85 annot_file.replace(pos, std::string(kImagesFolder).length(), std::string(kAnnotationsFolder));
86 }
87 // then replace the extensions. the image extension to annotation extension
88 if ((pos = annot_file.find(image_ext, 0)) != std::string::npos) {
89 annot_file.replace(pos, std::string(kAnnotExt).length(), std::string(kAnnotExt));
90 }
91 std::shared_ptr<Tensor> annotation_tensor;
92 // load annotations and log failure
93 if (!LoadAnnotationFile(annot_file, &annotation_tensor)) {
94 MS_LOG(ERROR) << "Loading Annotations failed in GetInputImagesAndAnnotations";
95 EXPECT_EQ(0, 1);
96 }
97 // load image
98 GetInputImage(image_file);
99 // add image and annotation to the tensor table
100 TensorRow row_data({std::move(input_tensor_), std::move(annotation_tensor)});
101 images_and_annotations_.push_back(row_data);
102 files_fetched++;
103 if (files_fetched == num_of_samples) {
104 break;
105 }
106 }
107 }
108
SaveImagesWithAnnotations(BBoxOpCommon::FileType type,const std::string & op_name,const TensorTable & table)109 void BBoxOpCommon::SaveImagesWithAnnotations(BBoxOpCommon::FileType type, const std::string &op_name,
110 const TensorTable &table) {
111 int i = 0;
112 for (auto &row : table) {
113 std::shared_ptr<Tensor> row_to_save;
114 Status swap_status = SwapRedAndBlue(row[0], &row_to_save);
115 if (!swap_status.IsOk()) {
116 MS_LOG(ERROR) << "Swapping red and blue channels failed in SaveImagesWithAnnotations.";
117 EXPECT_TRUE(swap_status.IsOk());
118 }
119 cv::Mat image = std::static_pointer_cast<CVTensor>(row_to_save)->mat();
120 uint32_t num_of_boxes = row[1]->shape()[0];
121 bool passing_data_fetch = true;
122 // For each bounding box draw on the image.
123 for (uint32_t i = 0; i < num_of_boxes; i++) {
124 float x = 0.0, y = 0.0, w = 0.0, h = 0.0;
125 passing_data_fetch &= row[1]->GetItemAt<float>(&x, {i, 0}).IsOk();
126 passing_data_fetch &= row[1]->GetItemAt<float>(&y, {i, 1}).IsOk();
127 passing_data_fetch &= row[1]->GetItemAt<float>(&w, {i, 2}).IsOk();
128 passing_data_fetch &= row[1]->GetItemAt<float>(&h, {i, 3}).IsOk();
129 if (!passing_data_fetch) {
130 MS_LOG(ERROR) << "Fetching bbox coordinates failed in SaveImagesWithAnnotations.";
131 EXPECT_TRUE(passing_data_fetch);
132 }
133 cv::Rect bbox(x, y, w, h);
134 cv::rectangle(image, bbox, cv::Scalar(255, 0, 0), 10, 8, 0);
135 }
136 bool im_write_success = false;
137 // if user wants to save an expected image, use the path to the source folder.
138 if (type == FileType::kExpected) {
139 im_write_success = cv::imwrite(
140 image_folder_src_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt), image);
141 } else {
142 // otherwise if we are saving actual images only for comparison, save in current working dir in build folders.
143 im_write_success =
144 cv::imwrite(std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt), image);
145 }
146 if (!im_write_success) {
147 MS_LOG(ERROR) << "Image write failed in SaveImagesWithAnnotations.";
148 EXPECT_TRUE(im_write_success);
149 }
150 i += 1;
151 }
152 }
153
CompareActualAndExpected(const std::string & op_name)154 void BBoxOpCommon::CompareActualAndExpected(const std::string &op_name) {
155 size_t num_of_images = images_and_annotations_.size();
156 for (size_t i = 0; i < num_of_images; i++) {
157 // load actual and expected images.
158 std::string actual_path = std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt);
159 std::string expected_path =
160 image_folder_build_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt);
161 cv::Mat expect_img = cv::imread(expected_path, cv::IMREAD_COLOR);
162 cv::Mat actual_img = cv::imread(actual_path, cv::IMREAD_COLOR);
163 // after comparison is done remove temporary file
164 EXPECT_TRUE(remove(actual_path.c_str()) == 0);
165 // compare using ==operator by Tensor
166 std::shared_ptr<CVTensor> expect_img_t, actual_img_t;
167 CVTensor::CreateFromMat(expect_img, 3, &expect_img_t);
168 CVTensor::CreateFromMat(actual_img, 3, &actual_img_t);
169 if (actual_img.data) {
170 EXPECT_EQ(*expect_img_t == *actual_img_t, true);
171 } else {
172 MS_LOG(ERROR) << "Not pass verification! Image data is null.";
173 EXPECT_EQ(0, 1);
174 }
175 }
176 }
177
LoadAnnotationFile(const std::string & path,std::shared_ptr<Tensor> * target_BBox)178 bool BBoxOpCommon::LoadAnnotationFile(const std::string &path, std::shared_ptr<Tensor> *target_BBox) {
179 if (!Path(path).Exists()) {
180 MS_LOG(ERROR) << "File is not found : " + path;
181 return false;
182 }
183 XMLDocument doc;
184 XMLError e = doc.LoadFile(mindspore::common::SafeCStr(path));
185 if (e != XMLError::XML_SUCCESS) {
186 MS_LOG(ERROR) << "Xml load failed";
187 return false;
188 }
189 XMLElement *root = doc.RootElement();
190 if (root == nullptr) {
191 MS_LOG(ERROR) << "Xml load root element error";
192 return false;
193 }
194 XMLElement *object = root->FirstChildElement("object");
195 if (object == nullptr) {
196 MS_LOG(ERROR) << "No object find in " + path;
197 return false;
198 }
199 std::vector<float> return_value_list;
200 dsize_t bbox_count = 0; // keep track of number of bboxes in file
201 dsize_t bbox_val_count = 4; // creating bboxes of size 4 to test function
202 // FILE OK TO READ
203 while (object != nullptr) {
204 bbox_count += 1;
205 std::string label_name;
206 float xmin = 0.0, ymin = 0.0, xmax = 0.0, ymax = 0.0;
207 XMLElement *bbox_node = object->FirstChildElement("bndbox");
208 if (bbox_node != nullptr) {
209 XMLElement *xmin_node = bbox_node->FirstChildElement("xmin");
210 if (xmin_node != nullptr) xmin = xmin_node->FloatText();
211 XMLElement *ymin_node = bbox_node->FirstChildElement("ymin");
212 if (ymin_node != nullptr) ymin = ymin_node->FloatText();
213 XMLElement *xmax_node = bbox_node->FirstChildElement("xmax");
214 if (xmax_node != nullptr) xmax = xmax_node->FloatText();
215 XMLElement *ymax_node = bbox_node->FirstChildElement("ymax");
216 if (ymax_node != nullptr) ymax = ymax_node->FloatText();
217 } else {
218 MS_LOG(ERROR) << "bndbox dismatch in " + path;
219 return false;
220 }
221 if (xmin > 0 && ymin > 0 && xmax > xmin && ymax > ymin) {
222 for (auto item : {xmin, ymin, xmax - xmin, ymax - ymin}) {
223 return_value_list.push_back(item);
224 }
225 }
226 object = object->NextSiblingElement("object"); // Read next BBox if exists
227 }
228 std::shared_ptr<Tensor> ret_value;
229 Status s = Tensor::CreateFromVector(return_value_list, TensorShape({bbox_count, bbox_val_count}), &ret_value);
230 EXPECT_TRUE(s.IsOk());
231 (*target_BBox) = ret_value; // load bbox from file into return
232 return true;
233 }
234