1 /**
2 * Copyright 2019-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <fstream>
17 #include <iostream>
18 #include <memory>
19 #include <string>
20
21 #include "common/common.h"
22 #include "utils/ms_utils.h"
23 #include "minddata/dataset/core/client.h"
24 #include "minddata/dataset/core/global_context.h"
25 #include "minddata/dataset/engine/datasetops/source/cifar_op.h"
26 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
27 #include "minddata/dataset/engine/datasetops/source/sampler/random_sampler.h"
28 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
29 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
30 #include "minddata/dataset/util/path.h"
31 #include "minddata/dataset/util/status.h"
32 #include "gtest/gtest.h"
33 #include "utils/log_adapter.h"
34 #include "securec.h"
35
36 namespace common = mindspore::common;
37
38 using namespace mindspore::dataset;
39 using mindspore::LogStream;
40 using mindspore::ExceptionType::NoExceptionType;
41 using mindspore::MsLogLevel::ERROR;
42
43 std::shared_ptr<ExecutionTree> Build(std::vector<std::shared_ptr<DatasetOp>> ops);
44
Cifarop(uint64_t num_works,uint64_t rows,uint64_t conns,std::string path,std::shared_ptr<SamplerRT> sampler=nullptr,bool cifar10=true)45 std::shared_ptr<CifarOp> Cifarop(uint64_t num_works, uint64_t rows, uint64_t conns, std::string path,
46 std::shared_ptr<SamplerRT> sampler = nullptr, bool cifar10 = true) {
47 std::shared_ptr<ConfigManager> cfg = GlobalContext::config_manager();
48 auto num_workers = cfg->num_parallel_workers();
49 std::string usage = "";
50 CifarOp::CifarType cifar_type;
51 if (cifar10) {
52 cifar_type = CifarOp::kCifar10;
53 } else {
54 cifar_type = CifarOp::kCifar100;
55 }
56 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
57 TensorShape scalar = TensorShape::CreateScalar();
58 (void)schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1));
59 if (cifar_type == CifarOp::kCifar10) {
60 (void)schema->AddColumn(ColDescriptor("label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar));
61 } else {
62 (void)schema->AddColumn(
63 ColDescriptor("coarse_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &scalar));
64 TensorShape another_scalar = TensorShape::CreateScalar();
65 (void)schema->AddColumn(
66 ColDescriptor("fine_label", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 0, &another_scalar));
67 }
68
69 if (sampler == nullptr) {
70 const int64_t num_samples = 0;
71 const int64_t start_index = 0;
72 sampler = std::make_shared<SequentialSamplerRT>(start_index, num_samples);
73 }
74
75 std::shared_ptr<CifarOp> so =
76 std::make_shared<CifarOp>(cifar_type, usage, num_workers, path, conns, std::move(schema), std::move(sampler));
77 return so;
78 }
79
80 class MindDataTestCifarOp : public UT::DatasetOpTesting {
81 protected:
82 };
83
TEST_F(MindDataTestCifarOp,TestSequentialSamplerCifar10)84 TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) {
85 // Note: CIFAR and Mnist datasets are not included
86 // as part of the build tree.
87 // Download datasets and rebuild if data doesn't
88 // appear in this dataset
89 // Example: python tests/dataset/data/prep_data.py
90 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
91 auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr)});
92 tree->Prepare();
93 Status rc = tree->Launch();
94 if (rc.IsError()) {
95 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << ".";
96 EXPECT_TRUE(false);
97 } else {
98 DatasetIterator di(tree);
99 TensorMap tensor_map;
100 ASSERT_OK(di.GetNextAsMap(&tensor_map));
101 EXPECT_TRUE(rc.IsOk());
102 uint64_t i = 0;
103 uint32_t label = 0;
104 // Note: only iterating first 100 rows then break out.
105 while (tensor_map.size() != 0 && i < 100) {
106 tensor_map["label"]->GetItemAt<uint32_t>(&label, {});
107 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n";
108 i++;
109 ASSERT_OK(di.GetNextAsMap(&tensor_map));
110 }
111 EXPECT_TRUE(i == 100);
112 }
113 }
114
TEST_F(MindDataTestCifarOp,TestRandomSamplerCifar10)115 TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) {
116 uint32_t original_seed = GlobalContext::config_manager()->seed();
117 GlobalContext::config_manager()->set_seed(0);
118 std::shared_ptr<SamplerRT> sampler = std::make_unique<RandomSamplerRT>(true, 12, true);
119 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
120 auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler))});
121 tree->Prepare();
122 Status rc = tree->Launch();
123 if (rc.IsError()) {
124 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << ".";
125 EXPECT_TRUE(false);
126 } else {
127 DatasetIterator di(tree);
128 TensorMap tensor_map;
129 ASSERT_OK(di.GetNextAsMap(&tensor_map));
130 EXPECT_TRUE(rc.IsOk());
131 uint64_t i = 0;
132 uint32_t label = 0;
133 while (tensor_map.size() != 0) {
134 tensor_map["label"]->GetItemAt<uint32_t>(&label, {});
135 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << "label:" << label << "\n";
136 i++;
137 ASSERT_OK(di.GetNextAsMap(&tensor_map));
138 }
139 EXPECT_TRUE(i == 12);
140 }
141 GlobalContext::config_manager()->set_seed(original_seed);
142 }
143
TEST_F(MindDataTestCifarOp,TestSequentialSamplerCifar100)144 TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar100) {
145 std::string folder_path = datasets_root_path_ + "/testCifar100Data/";
146 auto tree = Build({Cifarop(16, 2, 32, folder_path, nullptr, false)});
147 tree->Prepare();
148 Status rc = tree->Launch();
149 if (rc.IsError()) {
150 MS_LOG(ERROR) << "Return code error detected during tree launch: " << common::SafeCStr(rc.ToString()) << ".";
151 EXPECT_TRUE(false);
152 } else {
153 DatasetIterator di(tree);
154 TensorMap tensor_map;
155 ASSERT_OK(di.GetNextAsMap(&tensor_map));
156 EXPECT_TRUE(rc.IsOk());
157 uint64_t i = 0;
158 uint32_t coarse = 0;
159 uint32_t fine = 0;
160 // only iterate to 100 then break out of loop
161 while (tensor_map.size() != 0 && i < 100) {
162 tensor_map["coarse_label"]->GetItemAt<uint32_t>(&coarse, {});
163 tensor_map["fine_label"]->GetItemAt<uint32_t>(&fine, {});
164 MS_LOG(DEBUG) << "row: " << i << "\t" << tensor_map["image"]->shape() << " coarse:" << coarse << " fine:" << fine
165 << "\n";
166 i++;
167 ASSERT_OK(di.GetNextAsMap(&tensor_map));
168 }
169 EXPECT_TRUE(i == 100);
170 }
171 }
172