1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18
19 using namespace mindspore::dataset;
20 using mindspore::dataset::Tensor;
21
22 class MindDataTestPipeline : public UT::DatasetOpTesting {
23 protected:
24 };
25
TEST_F(MindDataTestPipeline,TestDIV2KBasic)26 TEST_F(MindDataTestPipeline, TestDIV2KBasic) {
27 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasic.";
28
29 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
30 std::string usage = "train"; // train valid, all
31 std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
32 int32_t scale = 2; // 2, 3, 4, 8
33
34 // Create a DIV2K Dataset
35 std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale);
36 EXPECT_NE(ds, nullptr);
37
38 // Create an iterator over the result of the above dataset
39 // This will trigger the creation of the Execution Tree and launch it.
40 std::shared_ptr<Iterator> iter = ds->CreateIterator();
41 EXPECT_NE(iter, nullptr);
42
43 // Iterate the dataset and get each row
44 std::unordered_map<std::string, mindspore::MSTensor> row;
45 ASSERT_OK(iter->GetNextRow(&row));
46
47 uint64_t i = 0;
48 while (row.size() != 0) {
49 i++;
50 auto hr_image = row["hr_image"];
51 auto lr_image = row["lr_image"];
52 MS_LOG(INFO) << "Tensor hr_image shape: " << hr_image.Shape();
53 MS_LOG(INFO) << "Tensor lr_image shape: " << lr_image.Shape();
54 ASSERT_OK(iter->GetNextRow(&row));
55 }
56
57 EXPECT_EQ(i, 5);
58
59 // Manually terminate the pipeline
60 iter->Stop();
61 }
62
TEST_F(MindDataTestPipeline,TestDIV2KBasicWithPipeline)63 TEST_F(MindDataTestPipeline, TestDIV2KBasicWithPipeline) {
64 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KBasicWithPipeline.";
65
66 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
67 std::string usage = "train"; // train valid, all
68 std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
69 int32_t scale = 2; // 2, 3, 4, 8
70
71 // Create two DIV2K Dataset
72 std::shared_ptr<Dataset> ds1 =
73 DIV2K(dataset_path, usage, downgrade, scale, false, std::make_shared<RandomSampler>(false, 2));
74 std::shared_ptr<Dataset> ds2 =
75 DIV2K(dataset_path, usage, downgrade, scale, false, std::make_shared<RandomSampler>(false, 3));
76 EXPECT_NE(ds1, nullptr);
77 EXPECT_NE(ds2, nullptr);
78
79 // Create two Repeat operation on ds
80 int32_t repeat_num = 3;
81 ds1 = ds1->Repeat(repeat_num);
82 EXPECT_NE(ds1, nullptr);
83 repeat_num = 2;
84 ds2 = ds2->Repeat(repeat_num);
85 EXPECT_NE(ds2, nullptr);
86
87 // Create two Project operation on ds
88 std::vector<std::string> column_project = {"hr_image", "lr_image"};
89 ds1 = ds1->Project(column_project);
90 EXPECT_NE(ds1, nullptr);
91 ds2 = ds2->Project(column_project);
92 EXPECT_NE(ds2, nullptr);
93
94 // Create a Concat operation on the ds
95 ds1 = ds1->Concat({ds2});
96 EXPECT_NE(ds1, nullptr);
97
98 // Create an iterator over the result of the above dataset
99 // This will trigger the creation of the Execution Tree and launch it.
100 std::shared_ptr<Iterator> iter = ds1->CreateIterator();
101 EXPECT_NE(iter, nullptr);
102
103 // Iterate the dataset and get each row
104 std::unordered_map<std::string, mindspore::MSTensor> row;
105 ASSERT_OK(iter->GetNextRow(&row));
106
107 uint64_t i = 0;
108 while (row.size() != 0) {
109 i++;
110 auto image = row["hr_image"];
111 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
112 ASSERT_OK(iter->GetNextRow(&row));
113 }
114
115 EXPECT_EQ(i, 12);
116
117 // Manually terminate the pipeline
118 iter->Stop();
119 }
120
TEST_F(MindDataTestPipeline,TestDIV2KGetters)121 TEST_F(MindDataTestPipeline, TestDIV2KGetters) {
122 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KGetters.";
123
124 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
125 std::string usage = "train"; // train valid, all
126 std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
127 int32_t scale = 2; // 2, 3, 4, 8
128
129 // Create a DIV2K Dataset
130 std::shared_ptr<Dataset> ds1 =
131 DIV2K(dataset_path, usage, downgrade, scale, false, std::make_shared<RandomSampler>(false, 2));
132 std::shared_ptr<Dataset> ds2 =
133 DIV2K(dataset_path, usage, downgrade, scale, false, std::make_shared<RandomSampler>(false, 3));
134 std::vector<std::string> column_names = {"hr_image", "lr_image"};
135
136 EXPECT_NE(ds1, nullptr);
137 EXPECT_EQ(ds1->GetDatasetSize(), 2);
138 EXPECT_EQ(ds1->GetColumnNames(), column_names);
139
140 EXPECT_NE(ds2, nullptr);
141 EXPECT_EQ(ds2->GetDatasetSize(), 3);
142 EXPECT_EQ(ds2->GetColumnNames(), column_names);
143 }
144
TEST_F(MindDataTestPipeline,TestDIV2KDecode)145 TEST_F(MindDataTestPipeline, TestDIV2KDecode) {
146 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KDecode.";
147
148 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
149 std::string usage = "train"; // train valid, all
150 std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
151 int32_t scale = 2; // 2, 3, 4, 8
152
153 // Create a DIV2K Dataset
154 std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale, true, std::make_shared<RandomSampler>());
155 EXPECT_NE(ds, nullptr);
156
157 // Create an iterator over the result of the above dataset
158 // This will trigger the creation of the Execution Tree and launch it.
159 std::shared_ptr<Iterator> iter = ds->CreateIterator();
160 EXPECT_NE(iter, nullptr);
161
162 // Iterate the dataset and get each row
163 std::unordered_map<std::string, mindspore::MSTensor> row;
164 ASSERT_OK(iter->GetNextRow(&row));
165
166 uint64_t i = 0;
167 while (row.size() != 0) {
168 i++;
169 auto hr_image = row["hr_image"];
170 auto lr_image = row["lr_image"];
171 auto h_size = hr_image.Shape().size();
172 auto l_size = lr_image.Shape().size();
173 MS_LOG(INFO) << "Tensor hr_image shape size: " << h_size;
174 MS_LOG(INFO) << "Tensor lr_image shape size: " << l_size;
175 EXPECT_GT(h_size, 1); // Verify decode=true took effect
176 EXPECT_GT(l_size, 1); // Verify decode=true took effect
177 ASSERT_OK(iter->GetNextRow(&row));
178 }
179
180 EXPECT_EQ(i, 5);
181
182 // Manually terminate the pipeline
183 iter->Stop();
184 }
185
TEST_F(MindDataTestPipeline,TestDIV2KNumSamplers)186 TEST_F(MindDataTestPipeline, TestDIV2KNumSamplers) {
187 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KNumSamplers.";
188
189 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
190 std::string usage = "train"; // train valid, all
191 std::string downgrade = "bicubic"; // bicubic, unknown, mild, difficult, wild
192 int32_t scale = 2; // 2, 3, 4, 8
193
194 // Create a DIV2K Dataset
195 std::shared_ptr<Dataset> ds =
196 DIV2K(dataset_path, usage, downgrade, scale, true, std::make_shared<SequentialSampler>(0, 1));
197 EXPECT_NE(ds, nullptr);
198
199 // Create an iterator over the result of the above dataset
200 // This will trigger the creation of the Execution Tree and launch it.
201 std::shared_ptr<Iterator> iter = ds->CreateIterator();
202 EXPECT_NE(iter, nullptr);
203
204 // Iterate the dataset and get each row
205 std::unordered_map<std::string, mindspore::MSTensor> row;
206 ASSERT_OK(iter->GetNextRow(&row));
207
208 uint64_t i = 0;
209 while (row.size() != 0) {
210 i++;
211 auto hr_image = row["hr_image"];
212 auto lr_image = row["lr_image"];
213
214 MS_LOG(INFO) << "Tensor hr_image shape: " << hr_image.Shape();
215 MS_LOG(INFO) << "Tensor lr_image shape: " << lr_image.Shape();
216
217 ASSERT_OK(iter->GetNextRow(&row));
218 }
219
220 EXPECT_EQ(i, 1);
221
222 // Manually terminate the pipeline
223 iter->Stop();
224 }
225
TEST_F(MindDataTestPipeline,TestDIV2KError)226 TEST_F(MindDataTestPipeline, TestDIV2KError) {
227 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KError.";
228
229 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
230 std::string usage = "train"; // train valid, all
231 std::string downgrade = "unknown"; // bicubic, unknown, mild, difficult, wild
232 int32_t scale = 2; // 2, 3, 4, 8
233
234 // Create a DIV2K Dataset with non-existing dataset dir
235 std::shared_ptr<Dataset> ds0 = DIV2K("NotExistFile", usage, downgrade, scale);
236 EXPECT_NE(ds0, nullptr);
237
238 // Create an iterator over the result of the above dataset
239 std::shared_ptr<Iterator> iter0 = ds0->CreateIterator();
240 // Expect failure: invalid DIV2K input
241 EXPECT_EQ(iter0, nullptr);
242
243 // Create a DIV2K Dataset with err usage
244 std::shared_ptr<Dataset> ds1 = DIV2K(dataset_path, "test", downgrade, scale);
245 EXPECT_NE(ds1, nullptr);
246
247 // Create an iterator over the result of the above dataset
248 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
249 // Expect failure: invalid DIV2K input
250 EXPECT_EQ(iter1, nullptr);
251
252 // Create a DIV2K Dataset with err scale
253 std::shared_ptr<Dataset> ds2 = DIV2K(dataset_path, usage, downgrade, 16);
254 EXPECT_NE(ds2, nullptr);
255
256 // Create an iterator over the result of the above dataset
257 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
258 // Expect failure: invalid DIV2K input
259 EXPECT_EQ(iter2, nullptr);
260
261 // Create a DIV2K Dataset with err downgrade
262 std::shared_ptr<Dataset> ds3 = DIV2K(dataset_path, usage, "downgrade", scale);
263 EXPECT_NE(ds3, nullptr);
264
265 // Create an iterator over the result of the above dataset
266 std::shared_ptr<Iterator> iter3 = ds3->CreateIterator();
267 // Expect failure: invalid DIV2K input
268 EXPECT_EQ(iter3, nullptr);
269
270 // Create a DIV2K Dataset with scale 8 and downgrade unknown
271 std::shared_ptr<Dataset> ds4 = DIV2K(dataset_path, usage, "unknown", 8);
272 EXPECT_NE(ds4, nullptr);
273
274 // Create an iterator over the result of the above dataset
275 std::shared_ptr<Iterator> iter4 = ds4->CreateIterator();
276 // Expect failure: invalid DIV2K input
277 EXPECT_EQ(iter4, nullptr);
278
279 // Create a DIV2K Dataset with scale 2 and downgrade mild
280 std::shared_ptr<Dataset> ds5 = DIV2K(dataset_path, usage, "mild", 2);
281 EXPECT_NE(ds5, nullptr);
282
283 // Create an iterator over the result of the above dataset
284 std::shared_ptr<Iterator> iter5 = ds5->CreateIterator();
285 // Expect failure: invalid DIV2K input
286 EXPECT_EQ(iter5, nullptr);
287 }
288
TEST_F(MindDataTestPipeline,TestDIV2KWithNullSamplerError)289 TEST_F(MindDataTestPipeline, TestDIV2KWithNullSamplerError) {
290 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDIV2KWithNullSamplerError.";
291
292 std::string dataset_path = datasets_root_path_ + "/testDIV2KData/div2k";
293 std::string usage = "train"; // train valid, all
294 int32_t scale = 2; // 2, 3, 4, 8
295 std::string downgrade = "unknown"; // bicubic, unknown, mild, difficult, wild
296
297 // Create a DIV2K Dataset
298 std::shared_ptr<Dataset> ds = DIV2K(dataset_path, usage, downgrade, scale, false, nullptr);
299 EXPECT_NE(ds, nullptr);
300
301 // Create an iterator over the result of the above dataset
302 std::shared_ptr<Iterator> iter = ds->CreateIterator();
303 // Expect failure: invalid DIV2K input, sampler cannot be nullptr
304 EXPECT_EQ(iter, nullptr);
305 }