1 /** 2 * Copyright 2020 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "bboxop_common.h" 18 19 #include <memory> 20 #include <string> 21 #include <vector> 22 #include <iostream> 23 24 #include <stdio.h> 25 26 #include "./tinyxml2.h" 27 #include "opencv2/opencv.hpp" 28 #include "utils/ms_utils.h" 29 #include "minddata/dataset/core/cv_tensor.h" 30 #include "minddata/dataset/util/path.h" 31 #include "minddata/dataset/include/dataset/constants.h" 32 #include "utils/log_adapter.h" 33 34 using namespace mindspore::dataset; 35 using namespace UT::CVOP::BBOXOP; 36 using tinyxml2::XMLDocument; 37 using tinyxml2::XMLElement; 38 using tinyxml2::XMLError; 39 40 const char kAnnotationsFolder[] = "/Annotations/"; 41 const char kImagesFolder[] = "/JPEGImages/"; 42 const char kExpectedName[] = "apple_expect_"; 43 const char kActualName[] = "Actual"; 44 const char kAnnotExt[] = ".xml"; 45 const char kImageExt[] = ".jpg"; 46 47 BBoxOpCommon::BBoxOpCommon() {} 48 49 BBoxOpCommon::~BBoxOpCommon() {} 50 51 void BBoxOpCommon::SetUp() { 52 MS_LOG(INFO) << "starting test."; 53 image_folder_build_ = "data/dataset/imagefolder/"; 54 image_folder_src_ = "../../../../../tests/ut/data/dataset/imagefolder/"; 55 std::string dir_path = "data/dataset/testVOC2012_2"; 56 GetInputImagesAndAnnotations(dir_path); 57 } 58 59 void BBoxOpCommon::GetInputImagesAndAnnotations(const std::string &dir, std::size_t num_of_samples) { 60 std::string images_path = dir + std::string(kImagesFolder); 61 std::string annots_path = dir + std::string(kAnnotationsFolder); 62 Path dir_path(images_path); 63 std::shared_ptr<Path::DirIterator> image_dir_itr = Path::DirIterator::OpenDirectory(&dir_path); 64 std::vector<std::string> paths_to_fetch; 65 if (!dir_path.Exists()) { 66 MS_LOG(ERROR) << "Images folder was not found : " + images_path; 67 EXPECT_TRUE(dir_path.Exists()); 68 } 69 // get image file paths 70 while (image_dir_itr->HasNext()) { 71 Path image_path = image_dir_itr->Next(); 72 if (image_path.Extension() == std::string(kImageExt)) { 73 paths_to_fetch.push_back(image_path.ToString()); 74 } 75 } 76 // sort fetched files 77 std::sort(paths_to_fetch.begin(), paths_to_fetch.end()); 78 std::size_t files_fetched = 0; 79 for (const auto &image_file : paths_to_fetch) { 80 std::string image_ext = std::string(kImageExt); 81 std::string annot_file = image_file; 82 std::size_t pos = 0; 83 // first replace the Image dir with the Annotation dir. 84 if ((pos = image_file.find(std::string(kImagesFolder), 0)) != std::string::npos) { 85 annot_file.replace(pos, std::string(kImagesFolder).length(), std::string(kAnnotationsFolder)); 86 } 87 // then replace the extensions. the image extension to annotation extension 88 if ((pos = annot_file.find(image_ext, 0)) != std::string::npos) { 89 annot_file.replace(pos, std::string(kAnnotExt).length(), std::string(kAnnotExt)); 90 } 91 std::shared_ptr<Tensor> annotation_tensor; 92 // load annotations and log failure 93 if (!LoadAnnotationFile(annot_file, &annotation_tensor)) { 94 MS_LOG(ERROR) << "Loading Annotations failed in GetInputImagesAndAnnotations"; 95 EXPECT_EQ(0, 1); 96 } 97 // load image 98 GetInputImage(image_file); 99 // add image and annotation to the tensor table 100 TensorRow row_data({std::move(input_tensor_), std::move(annotation_tensor)}); 101 images_and_annotations_.push_back(row_data); 102 files_fetched++; 103 if (files_fetched == num_of_samples) { 104 break; 105 } 106 } 107 } 108 109 void BBoxOpCommon::SaveImagesWithAnnotations(BBoxOpCommon::FileType type, const std::string &op_name, 110 const TensorTable &table) { 111 int i = 0; 112 for (auto &row : table) { 113 std::shared_ptr<Tensor> row_to_save; 114 Status swap_status = SwapRedAndBlue(row[0], &row_to_save); 115 if (!swap_status.IsOk()) { 116 MS_LOG(ERROR) << "Swapping red and blue channels failed in SaveImagesWithAnnotations."; 117 EXPECT_TRUE(swap_status.IsOk()); 118 } 119 cv::Mat image = std::static_pointer_cast<CVTensor>(row_to_save)->mat(); 120 uint32_t num_of_boxes = row[1]->shape()[0]; 121 bool passing_data_fetch = true; 122 // For each bounding box draw on the image. 123 for (uint32_t i = 0; i < num_of_boxes; i++) { 124 float x = 0.0, y = 0.0, w = 0.0, h = 0.0; 125 passing_data_fetch &= row[1]->GetItemAt<float>(&x, {i, 0}).IsOk(); 126 passing_data_fetch &= row[1]->GetItemAt<float>(&y, {i, 1}).IsOk(); 127 passing_data_fetch &= row[1]->GetItemAt<float>(&w, {i, 2}).IsOk(); 128 passing_data_fetch &= row[1]->GetItemAt<float>(&h, {i, 3}).IsOk(); 129 if (!passing_data_fetch) { 130 MS_LOG(ERROR) << "Fetching bbox coordinates failed in SaveImagesWithAnnotations."; 131 EXPECT_TRUE(passing_data_fetch); 132 } 133 cv::Rect bbox(x, y, w, h); 134 cv::rectangle(image, bbox, cv::Scalar(255, 0, 0), 10, 8, 0); 135 } 136 bool im_write_success = false; 137 // if user wants to save an expected image, use the path to the source folder. 138 if (type == FileType::kExpected) { 139 im_write_success = cv::imwrite( 140 image_folder_src_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt), image); 141 } else { 142 // otherwise if we are saving actual images only for comparison, save in current working dir in build folders. 143 im_write_success = 144 cv::imwrite(std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt), image); 145 } 146 if (!im_write_success) { 147 MS_LOG(ERROR) << "Image write failed in SaveImagesWithAnnotations."; 148 EXPECT_TRUE(im_write_success); 149 } 150 i += 1; 151 } 152 } 153 154 void BBoxOpCommon::CompareActualAndExpected(const std::string &op_name) { 155 size_t num_of_images = images_and_annotations_.size(); 156 for (size_t i = 0; i < num_of_images; i++) { 157 // load actual and expected images. 158 std::string actual_path = std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt); 159 std::string expected_path = 160 image_folder_build_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt); 161 cv::Mat expect_img = cv::imread(expected_path, cv::IMREAD_COLOR); 162 cv::Mat actual_img = cv::imread(actual_path, cv::IMREAD_COLOR); 163 // after comparison is done remove temporary file 164 EXPECT_TRUE(remove(actual_path.c_str()) == 0); 165 // compare using ==operator by Tensor 166 std::shared_ptr<CVTensor> expect_img_t, actual_img_t; 167 CVTensor::CreateFromMat(expect_img, 3, &expect_img_t); 168 CVTensor::CreateFromMat(actual_img, 3, &actual_img_t); 169 if (actual_img.data) { 170 EXPECT_EQ(*expect_img_t == *actual_img_t, true); 171 } else { 172 MS_LOG(ERROR) << "Not pass verification! Image data is null."; 173 EXPECT_EQ(0, 1); 174 } 175 } 176 } 177 178 bool BBoxOpCommon::LoadAnnotationFile(const std::string &path, std::shared_ptr<Tensor> *target_BBox) { 179 if (!Path(path).Exists()) { 180 MS_LOG(ERROR) << "File is not found : " + path; 181 return false; 182 } 183 XMLDocument doc; 184 XMLError e = doc.LoadFile(mindspore::common::SafeCStr(path)); 185 if (e != XMLError::XML_SUCCESS) { 186 MS_LOG(ERROR) << "Xml load failed"; 187 return false; 188 } 189 XMLElement *root = doc.RootElement(); 190 if (root == nullptr) { 191 MS_LOG(ERROR) << "Xml load root element error"; 192 return false; 193 } 194 XMLElement *object = root->FirstChildElement("object"); 195 if (object == nullptr) { 196 MS_LOG(ERROR) << "No object find in " + path; 197 return false; 198 } 199 std::vector<float> return_value_list; 200 dsize_t bbox_count = 0; // keep track of number of bboxes in file 201 dsize_t bbox_val_count = 4; // creating bboxes of size 4 to test function 202 // FILE OK TO READ 203 while (object != nullptr) { 204 bbox_count += 1; 205 std::string label_name; 206 float xmin = 0.0, ymin = 0.0, xmax = 0.0, ymax = 0.0; 207 XMLElement *bbox_node = object->FirstChildElement("bndbox"); 208 if (bbox_node != nullptr) { 209 XMLElement *xmin_node = bbox_node->FirstChildElement("xmin"); 210 if (xmin_node != nullptr) xmin = xmin_node->FloatText(); 211 XMLElement *ymin_node = bbox_node->FirstChildElement("ymin"); 212 if (ymin_node != nullptr) ymin = ymin_node->FloatText(); 213 XMLElement *xmax_node = bbox_node->FirstChildElement("xmax"); 214 if (xmax_node != nullptr) xmax = xmax_node->FloatText(); 215 XMLElement *ymax_node = bbox_node->FirstChildElement("ymax"); 216 if (ymax_node != nullptr) ymax = ymax_node->FloatText(); 217 } else { 218 MS_LOG(ERROR) << "bndbox dismatch in " + path; 219 return false; 220 } 221 if (xmin > 0 && ymin > 0 && xmax > xmin && ymax > ymin) { 222 for (auto item : {xmin, ymin, xmax - xmin, ymax - ymin}) { 223 return_value_list.push_back(item); 224 } 225 } 226 object = object->NextSiblingElement("object"); // Read next BBox if exists 227 } 228 std::shared_ptr<Tensor> ret_value; 229 Status s = Tensor::CreateFromVector(return_value_list, TensorShape({bbox_count, bbox_val_count}), &ret_value); 230 EXPECT_TRUE(s.IsOk()); 231 (*target_BBox) = ret_value; // load bbox from file into return 232 return true; 233 } 234