1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18
19 using namespace mindspore::dataset;
20
21 class MindDataTestPipeline : public UT::DatasetOpTesting {
22 protected:
23 };
24
TEST_F(MindDataTestPipeline,TestIteratorEmptyColumn)25 TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
26 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn.";
27 // Create a Cifar10 Dataset
28 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
29 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 5));
30 EXPECT_NE(ds, nullptr);
31
32 // Create a Rename operation on ds
33 ds = ds->Rename({"image", "label"}, {"col1", "col2"});
34 EXPECT_NE(ds, nullptr);
35
36 // No columns are specified, use all columns
37 std::shared_ptr<Iterator> iter = ds->CreateIterator();
38 EXPECT_NE(iter, nullptr);
39
40 // Iterate the dataset and get each row
41 std::vector<mindspore::MSTensor> row;
42 ASSERT_OK(iter->GetNextRow(&row));
43 std::vector<int64_t> expect_image = {32, 32, 3};
44 std::vector<int64_t> expect_label = {};
45
46 uint64_t i = 0;
47 while (row.size() != 0) {
48 MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
49 EXPECT_EQ(expect_image, row[0].Shape());
50 EXPECT_EQ(expect_label, row[1].Shape());
51
52 ASSERT_OK(iter->GetNextRow(&row));
53 i++;
54 }
55
56 EXPECT_EQ(i, 5);
57
58 // Manually terminate the pipeline
59 iter->Stop();
60 }
61
TEST_F(MindDataTestPipeline,TestIteratorOneColumn)62 TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
63 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
64 // Create a Mnist Dataset
65 std::string folder_path = datasets_root_path_ + "/testMnistData/";
66 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
67 EXPECT_NE(ds, nullptr);
68
69 // Create a Batch operation on ds
70 int32_t batch_size = 2;
71 ds = ds->Batch(batch_size);
72 EXPECT_NE(ds, nullptr);
73
74 // Create an iterator over the result of the above dataset
75 // Only select "image" column and drop others
76 std::vector<std::string> columns = {"image"};
77 std::shared_ptr<Iterator> iter = ds->CreateIterator(columns, -1);
78 EXPECT_NE(iter, nullptr);
79
80 // Iterate the dataset and get each row
81 std::vector<mindspore::MSTensor> row;
82 ASSERT_OK(iter->GetNextRow(&row));
83 std::vector<int64_t> expect_image = {2, 28, 28, 1};
84
85 uint64_t i = 0;
86 while (row.size() != 0) {
87 for (auto &v : row) {
88 MS_LOG(INFO) << "image shape:" << v.Shape();
89 EXPECT_EQ(expect_image, v.Shape());
90 }
91 ASSERT_OK(iter->GetNextRow(&row));
92 i++;
93 }
94
95 EXPECT_EQ(i, 2);
96
97 // Manually terminate the pipeline
98 iter->Stop();
99 }
100
TEST_F(MindDataTestPipeline,TestIteratorReOrder)101 TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
102 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder.";
103 // Create a Cifar10 Dataset
104 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
105 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<SequentialSampler>(false, 4));
106 EXPECT_NE(ds, nullptr);
107
108 // Create a Take operation on ds
109 ds = ds->Take(2);
110 EXPECT_NE(ds, nullptr);
111
112 // Create an iterator over the result of the above dataset
113 // Reorder "image" and "label" column
114 std::vector<std::string> columns = {"label", "image"};
115 std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
116 EXPECT_NE(iter, nullptr);
117
118 // Iterate the dataset and get each row
119 std::vector<mindspore::MSTensor> row;
120 ASSERT_OK(iter->GetNextRow(&row));
121 std::vector<int64_t> expect_image = {32, 32, 3};
122 std::vector<int64_t> expect_label = {};
123
124 // Check "label" before "image" in row
125 uint64_t i = 0;
126 while (row.size() != 0) {
127 MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
128 EXPECT_EQ(expect_label, row[0].Shape());
129 EXPECT_EQ(expect_image, row[1].Shape());
130 ASSERT_OK(iter->GetNextRow(&row));
131 i++;
132 }
133
134 EXPECT_EQ(i, 2);
135
136 // Manually terminate the pipeline
137 iter->Stop();
138 }
139
TEST_F(MindDataTestPipeline,TestIteratorTwoColumns)140 TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
141 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns.";
142 // Create a VOC Dataset
143 std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
144 std::shared_ptr<Dataset> ds =
145 VOC(folder_path, "Detection", "train", {}, false, std::make_shared<SequentialSampler>(0, 4));
146 EXPECT_NE(ds, nullptr);
147
148 // Create a Repeat operation on ds
149 int32_t repeat_num = 2;
150 ds = ds->Repeat(repeat_num);
151 EXPECT_NE(ds, nullptr);
152
153 // Create an iterator over the result of the above dataset
154 // Only select "image" and "bbox" column
155 std::vector<std::string> columns = {"image", "bbox"};
156 std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
157 EXPECT_NE(iter, nullptr);
158
159 // Iterate the dataset and get each row
160 std::vector<mindspore::MSTensor> row;
161 ASSERT_OK(iter->GetNextRow(&row));
162 std::vector<std::vector<int64_t>> expect = {{173673}, {1, 4}, {173673}, {1, 4},
163 {147025}, {1, 4}, {211653}, {1, 4}};
164
165 uint64_t i = 0;
166 uint64_t j = 0;
167 while (row.size() != 0) {
168 MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
169 EXPECT_EQ(2, row.size());
170 EXPECT_EQ(expect[j++], row[0].Shape());
171 EXPECT_EQ(expect[j++], row[1].Shape());
172 ASSERT_OK(iter->GetNextRow(&row));
173 i++;
174 j = (j == expect.size()) ? 0 : j;
175 }
176
177 EXPECT_EQ(i, 8);
178
179 // Manually terminate the pipeline
180 iter->Stop();
181 }
182
TEST_F(MindDataTestPipeline,TestIteratorWrongColumn)183 TEST_F(MindDataTestPipeline, TestIteratorWrongColumn) {
184 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn.";
185 // Create a Mnist Dataset
186 std::string folder_path = datasets_root_path_ + "/testMnistData/";
187 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<RandomSampler>(false, 4));
188 EXPECT_NE(ds, nullptr);
189
190 // Pass wrong column name
191 std::vector<std::string> columns = {"digital"};
192 std::shared_ptr<Iterator> iter = ds->CreateIterator(columns);
193 EXPECT_EQ(iter, nullptr);
194 }
195
TEST_F(MindDataTestPipeline,TestIteratorNumEpoch)196 TEST_F(MindDataTestPipeline, TestIteratorNumEpoch) {
197 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorNumEpoch.";
198
199 std::shared_ptr<SchemaObj> schema = Schema();
200 int32_t random_data_num_row = 2;
201 int32_t num_epochs = 3;
202 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
203 std::shared_ptr<Dataset> ds = RandomData(random_data_num_row, schema)->SetNumWorkers(1);
204
205 std::shared_ptr<Iterator> iter = ds->CreateIterator({}, num_epochs);
206 ASSERT_NE(iter, nullptr); // should terminate test case if iterator is null
207 std::unordered_map<std::string, mindspore::MSTensor> row;
208
209 int32_t inner_row_cnt = 0;
210 int32_t total_row_cnt = 0;
211 for (int32_t i = 0; i < num_epochs; i++) {
212 ASSERT_OK(iter->GetNextRow(&row));
213 inner_row_cnt = 0;
214 while (row.size() != 0) {
215 ASSERT_OK(iter->GetNextRow(&row));
216 ++inner_row_cnt;
217 ++total_row_cnt;
218 }
219 EXPECT_EQ(inner_row_cnt, random_data_num_row);
220 }
221 EXPECT_EQ(total_row_cnt, random_data_num_row * num_epochs);
222 // This will go beyond the random_data_num_row*num_epoch limit, hence error code is expected
223 EXPECT_ERROR(iter->GetNextRow(&row));
224 // Manually terminate the pipeline
225 iter->Stop();
226 }
227
TEST_F(MindDataTestPipeline,TestIteratorNumEpochFail)228 TEST_F(MindDataTestPipeline, TestIteratorNumEpochFail) {
229 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorNumEpochFail.";
230
231 std::shared_ptr<SchemaObj> schema = Schema();
232 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
233 std::shared_ptr<Dataset> ds = RandomData(3, schema)->SetNumWorkers(1);
234 // expect nullptr due to incorrect num_epochs value.
235 EXPECT_EQ(ds->CreateIterator({}, 0), nullptr);
236 EXPECT_EQ(ds->CreateIterator({}, -2), nullptr);
237 }
238