1 /** 2 * Copyright 2019-2021 Huawei Technologies Co., Ltd 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include <fstream> 17 #include <iostream> 18 #include <memory> 19 #include <string> 20 21 #include "common/common.h" 22 #include "utils/ms_utils.h" 23 #include "minddata/dataset/core/client.h" 24 #include "minddata/dataset/core/global_context.h" 25 #include "minddata/dataset/engine/datasetops/source/cifar_op.h" 26 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" 27 #include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h" 28 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" 29 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h" 30 #include "minddata/dataset/util/path.h" 31 #include "minddata/dataset/util/status.h" 32 #include "gtest/gtest.h" 33 #include "utils/log_adapter.h" 34 #include "securec.h" 35 36 namespace common = mindspore::common; 37 38 using namespace mindspore::dataset; 39 using mindspore::LogStream; 40 using mindspore::ExceptionType::NoExceptionType; 41 using mindspore::MsLogLevel::ERROR; 42 43 std::shared_ptr<ExecutionTree> Build(std::vector<std::shared_ptr<DatasetOp>> ops); 44 45 std::shared_ptr<CifarOp> Cifarop(uint64_t num_works, uint64_t rows, uint64_t conns, std::string path, 46 std::shared_ptr<SamplerRT> sampler = nullptr, bool cifar10 = true) { 47 std::shared_ptr<ConfigManager> cfg = GlobalContext::config_manager(); 48 auto num_workers = cfg->num_parallel_workers(); 49 std::string usage = ""; 50 CifarOp::CifarType cifar_type; 51 if (cifar10) { 52 cifar_type = CifarOp::kCifar10; 53 } else { 54 cifar_type = CifarOp::kCifar100; 55 } 56 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); 57 TensorShape scalar = TensorShape::CreateScalar(); 58 (void)schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)); 59 if (cifar_type == CifarOp::kCifar10) { 60 (void)schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar)); 61 } else { 62 (void)schema->AddColumn( 63 ColDescriptor("coarse_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar)); 64 TensorShape another_scalar = TensorShape::CreateScalar(); 65 (void)schema->AddColumn( 66 ColDescriptor("fine_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &another_scalar)); 67 } 68 69 if (sampler == nullptr) { 70 const int64_t num_samples = 0; 71 const int64_t start_index = 0; 72 sampler = std::make_shared<SequentialSamplerRT>(start_index, num_samples); 73 } 74 75 std::shared_ptr<CifarOp> so = 76 std::make_shared<CifarOp>(cifar_type, usage, num_workers, path, conns, std::move(schema), std::move(sampler)); 77 return so; 78 } 79 80 class MindDataTestCifarOp : public UT::DatasetOpTesting { 81 protected: 82 }; 83 84 TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) { 85 // Note: CIFAR and Mnist datasets are not included 86 // as part of the build tree. 87 // Download datasets and rebuild if data doesn't 88 // appear in this dataset 89 // Example: python tests/dataset/data/prep_data.py 90 std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; 91 auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr)}); 92 tree->Prepare(); 93 Status rc = tree->Launch(); 94 if (rc.IsError()) { 95 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; 96 EXPECT_TRUE(false); 97 } else { 98 DatasetIterator di(tree); 99 TensorMap tensor_map; 100 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 101 EXPECT_TRUE(rc.IsOk()); 102 uint64_t i = 0; 103 uint32_t label = 0; 104 // Note: only iterating first 100 rows then break out. 105 while (tensor_map.size() != 0 && i < 100) { 106 tensor_map["label"]->GetItemAt<uint32_t>(&label, {}); 107 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; 108 i++; 109 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 110 } 111 EXPECT_TRUE(i == 100); 112 } 113 } 114 115 TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) { 116 uint32_t original_seed = GlobalContext::config_manager()->seed(); 117 GlobalContext::config_manager()->set_seed(0); 118 std::shared_ptr<SamplerRT> sampler = std::make_unique<RandomSamplerRT>(true, 12, true); 119 std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; 120 auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler))}); 121 tree->Prepare(); 122 Status rc = tree->Launch(); 123 if (rc.IsError()) { 124 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; 125 EXPECT_TRUE(false); 126 } else { 127 DatasetIterator di(tree); 128 TensorMap tensor_map; 129 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 130 EXPECT_TRUE(rc.IsOk()); 131 uint64_t i = 0; 132 uint32_t label = 0; 133 while (tensor_map.size() != 0) { 134 tensor_map["label"]->GetItemAt<uint32_t>(&label, {}); 135 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n"; 136 i++; 137 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 138 } 139 EXPECT_TRUE(i == 12); 140 } 141 GlobalContext::config_manager()->set_seed(original_seed); 142 } 143 144 TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar100) { 145 std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; 146 auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr, false)}); 147 tree->Prepare(); 148 Status rc = tree->Launch(); 149 if (rc.IsError()) { 150 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << "."; 151 EXPECT_TRUE(false); 152 } else { 153 DatasetIterator di(tree); 154 TensorMap tensor_map; 155 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 156 EXPECT_TRUE(rc.IsOk()); 157 uint64_t i = 0; 158 uint32_t coarse = 0; 159 uint32_t fine = 0; 160 // only iterate to 100 then break out of loop 161 while (tensor_map.size() != 0 && i < 100) { 162 tensor_map["coarse_label"]->GetItemAt<uint32_t>(&coarse, {}); 163 tensor_map["fine_label"]->GetItemAt<uint32_t>(&fine, {}); 164 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << " coarse:" << coarse << " fine:" << fine 165 << "\n"; 166 i++; 167 ASSERT_OK(di.GetNextAsMap(&tensor_map)); 168 } 169 EXPECT_TRUE(i == 100); 170 } 171 } 172