1 /** 2 * Copyright 2020-2021 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include <memory> 17 #include <string> 18 19 #include "common/common.h" 20 #include "minddata/dataset/core/client.h" 21 #include "minddata/dataset/engine/datasetops/source/celeba_op.h" 22 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" 23 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h" 24 #include "minddata/dataset/util/status.h" 25 #include "gtest/gtest.h" 26 #include "utils/log_adapter.h" 27 #include "securec.h" 28 29 using namespace mindspore::dataset; 30 using mindspore::LogStream; 31 using mindspore::ExceptionType::NoExceptionType; 32 using mindspore::MsLogLevel::ERROR; 33 34 // std::shared_ptr<RepeatOp> Repeat(int repeat_cnt); 35 36 // std::shared_ptr<ExecutionTree> Build(std::vector<std::shared_ptr<DatasetOp>> ops); 37 38 std::shared_ptr<CelebAOp> Celeba(int32_t num_workers, int32_t queue_size, const std::string &dir, 39 std::shared_ptr<SamplerRT> sampler = nullptr, bool decode = false, 40 const std::string &dataset_type = "all") { 41 if (sampler == nullptr) { 42 const int64_t num_samples = 0; 43 const int64_t start_index = 0; 44 sampler = std::make_shared<SequentialSamplerRT>(start_index, num_samples); 45 } 46 47 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); 48 (void)schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)); 49 (void)schema->AddColumn(ColDescriptor("attr", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 1)); 50 51 std::shared_ptr<ConfigManager> config_manager = GlobalContext::config_manager(); 52 auto op_connector_size = config_manager->op_connector_size(); 53 std::set<std::string> extensions = {}; 54 std::shared_ptr<CelebAOp> so = std::make_shared<CelebAOp>(num_workers, dir, op_connector_size, decode, dataset_type, 55 extensions, std::move(schema), std::move(sampler)); 56 return so; 57 } 58 59 class MindDataTestCelebaDataset : public UT::DatasetOpTesting { 60 protected: 61 }; 62 63 TEST_F(MindDataTestCelebaDataset, TestSequentialCeleba) { 64 std::string dir = datasets_root_path_ + "/testCelebAData/"; 65 uint32_t expect_labels[4][40] = {{0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 66 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}, 67 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 68 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 69 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 70 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 71 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 72 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}}; 73 uint32_t count = 0; 74 auto tree = Build({Celeba(16, 2, dir)}); 75 tree->Prepare(); 76 Status rc = tree->Launch(); 77 if (rc.IsError()) { 78 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; 79 EXPECT_TRUE(false); 80 } else { 81 DatasetIterator di(tree); 82 TensorMap tensor_map; 83 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 84 EXPECT_TRUE(rc.IsOk()); 85 while (tensor_map.size() != 0) { 86 uint32_t label; 87 for (int index = 0; index < 40; index++) { 88 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index}); 89 EXPECT_TRUE(expect_labels[count][index] == label); 90 } 91 count++; 92 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 93 } 94 EXPECT_TRUE(count == 4); 95 } 96 } 97 98 TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) { 99 std::string dir = datasets_root_path_ + "/testCelebAData/"; 100 uint32_t expect_labels[8][40] = {{0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 101 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}, 102 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 103 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 104 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 105 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 106 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 107 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}, 108 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 109 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}, 110 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 111 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 112 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 113 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}, 114 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 115 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}}; 116 uint32_t count = 0; 117 auto op1 = Celeba(16, 2, dir); 118 auto op2 = Repeat(2); 119 auto tree = Build({op1, op2}); 120 op1->SetTotalRepeats(2); 121 op1->SetNumRepeatsPerEpoch(2); 122 tree->Prepare(); 123 Status rc = tree->Launch(); 124 if (rc.IsError()) { 125 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; 126 EXPECT_TRUE(false); 127 } else { 128 DatasetIterator di(tree); 129 TensorMap tensor_map; 130 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 131 EXPECT_TRUE(rc.IsOk()); 132 while (tensor_map.size() != 0) { 133 uint32_t label; 134 for (int index = 0; index < 40; index++) { 135 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index}); 136 EXPECT_TRUE(expect_labels[count][index] == label); 137 } 138 count++; 139 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 140 } 141 EXPECT_TRUE(count == 8); 142 } 143 } 144 145 TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) { 146 std::vector<int64_t> indices({1}); 147 int64_t num_samples = 0; 148 std::shared_ptr<SamplerRT> sampler = std::make_shared<SubsetRandomSamplerRT>(indices, num_samples); 149 uint32_t expect_labels[1][40] = {{0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 150 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}}; 151 std::string dir = datasets_root_path_ + "/testCelebAData/"; 152 uint32_t count = 0; 153 auto tree = Build({Celeba(16, 2, dir, std::move(sampler))}); 154 tree->Prepare(); 155 Status rc = tree->Launch(); 156 if (rc.IsError()) { 157 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << "."; 158 EXPECT_TRUE(false); 159 } else { 160 DatasetIterator di(tree); 161 TensorMap tensor_map; 162 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 163 EXPECT_TRUE(rc.IsOk()); 164 while (tensor_map.size() != 0) { 165 uint32_t label; 166 for (int index = 0; index < 40; index++) { 167 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index}); 168 EXPECT_TRUE(expect_labels[count][index] == label); 169 } 170 count++; 171 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 172 } 173 EXPECT_TRUE(count == 1); 174 } 175 } 176