1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18 #include "minddata/dataset/include/dataset/transforms.h"
19 #include "minddata/dataset/include/dataset/vision.h"
20
21 using namespace mindspore::dataset;
22 using mindspore::dataset::BorderType;
23
24 class MindDataTestPipeline : public UT::DatasetOpTesting {
25 protected:
26 };
27
28 // Tests for vision C++ API A to Q TensorTransform Operations (in alphabetical order)
29
TEST_F(MindDataTestPipeline,TestAdjustGamma3Channel)30 TEST_F(MindDataTestPipeline, TestAdjustGamma3Channel) {
31 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma3Channel.";
32 std::string MindDataPath = "data/dataset";
33 std::string folder_path = MindDataPath + "/testImageNetData/train/";
34 std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
35 EXPECT_NE(ds1, nullptr);
36 std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
37 EXPECT_NE(ds2, nullptr);
38
39 auto adjustgamma_op = vision::AdjustGamma(10.0);
40
41 ds1 = ds1->Map({adjustgamma_op});
42 EXPECT_NE(ds1, nullptr);
43
44 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
45 EXPECT_NE(iter1, nullptr);
46 std::unordered_map<std::string, mindspore::MSTensor> row1;
47 iter1->GetNextRow(&row1);
48
49 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
50 EXPECT_NE(iter2, nullptr);
51 std::unordered_map<std::string, mindspore::MSTensor> row2;
52 iter2->GetNextRow(&row2);
53
54 uint64_t i = 0;
55 while (row1.size() != 0) {
56 i++;
57 auto image = row1["image"];
58 iter1->GetNextRow(&row1);
59 iter2->GetNextRow(&row2);
60 }
61 EXPECT_EQ(i, 2);
62
63 iter1->Stop();
64 iter2->Stop();
65 }
66
TEST_F(MindDataTestPipeline,TestAdjustGamma1Channel)67 TEST_F(MindDataTestPipeline, TestAdjustGamma1Channel) {
68 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma1Channel.";
69 std::string MindDataPath = "data/dataset";
70 std::string folder_path = MindDataPath + "/testImageNetData/train/";
71 std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
72 EXPECT_NE(ds1, nullptr);
73 std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
74 EXPECT_NE(ds2, nullptr);
75
76 auto adjustgamma_op = vision::AdjustGamma(10.0);
77 auto rgb2gray_op = vision::RGB2GRAY();
78
79 ds1 = ds1->Map({rgb2gray_op, adjustgamma_op});
80 EXPECT_NE(ds1, nullptr);
81
82 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
83 EXPECT_NE(iter1, nullptr);
84 std::unordered_map<std::string, mindspore::MSTensor> row1;
85 iter1->GetNextRow(&row1);
86
87 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
88 EXPECT_NE(iter2, nullptr);
89 std::unordered_map<std::string, mindspore::MSTensor> row2;
90 iter2->GetNextRow(&row2);
91
92 uint64_t i = 0;
93 while (row1.size() != 0) {
94 i++;
95 auto image = row1["image"];
96 iter1->GetNextRow(&row1);
97 iter2->GetNextRow(&row2);
98 }
99 EXPECT_EQ(i, 2);
100
101 iter1->Stop();
102 iter2->Stop();
103 }
104
TEST_F(MindDataTestPipeline,TestAdjustGammaParamCheck)105 TEST_F(MindDataTestPipeline, TestAdjustGammaParamCheck) {
106 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGammaParamCheck.";
107 std::string MindDataPath = "data/dataset";
108 std::string folder_path = MindDataPath + "/testImageNetData/train/";
109 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
110 EXPECT_NE(ds, nullptr);
111
112 // Case 1: Negative gamma
113 // Create objects for the tensor ops
114 std::shared_ptr<TensorTransform> adjust_gamma(new vision::AdjustGamma(-1, 1.0));
115 auto ds1 = ds->Map({adjust_gamma});
116 EXPECT_NE(ds1, nullptr);
117 // Create an iterator over the result of the above dataset
118 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
119 // Expect failure: invalid value of AdjustGamma
120 EXPECT_EQ(iter1, nullptr);
121 }
122
TEST_F(MindDataTestPipeline,TestAutoContrastSuccess1)123 TEST_F(MindDataTestPipeline, TestAutoContrastSuccess1) {
124 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAutoContrastSuccess1.";
125
126 // Create an ImageFolder Dataset
127 std::string folder_path = datasets_root_path_ + "/testPK/data/";
128 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
129 EXPECT_NE(ds, nullptr);
130
131 // Create a Repeat operation on ds
132 int32_t repeat_num = 3;
133 ds = ds->Repeat(repeat_num);
134 EXPECT_NE(ds, nullptr);
135
136 // Create auto contrast object with default values
137 std::shared_ptr<TensorTransform> auto_contrast(new vision::AutoContrast());
138 // Note: No need to check for output after calling API class constructor
139
140 // Create a Map operation on ds
141 ds = ds->Map({auto_contrast});
142 EXPECT_NE(ds, nullptr);
143
144 // Create a Batch operation on ds
145 int32_t batch_size = 1;
146 ds = ds->Batch(batch_size);
147 EXPECT_NE(ds, nullptr);
148
149 // Create an iterator over the result of the above dataset
150 // This will trigger the creation of the Execution Tree and launch it.
151 std::shared_ptr<Iterator> iter = ds->CreateIterator();
152 EXPECT_NE(iter, nullptr);
153
154 // Iterate the dataset and get each row
155 std::unordered_map<std::string, mindspore::MSTensor> row;
156 ASSERT_OK(iter->GetNextRow(&row));
157
158 uint64_t i = 0;
159 while (row.size() != 0) {
160 i++;
161 auto image = row["image"];
162 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
163 ASSERT_OK(iter->GetNextRow(&row));
164 }
165
166 EXPECT_EQ(i, 15);
167
168 // Manually terminate the pipeline
169 iter->Stop();
170 }
171
TEST_F(MindDataTestPipeline,TestAutoContrastSuccess2)172 TEST_F(MindDataTestPipeline, TestAutoContrastSuccess2) {
173 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAutoContrastSuccess2.";
174
175 // Create an ImageFolder Dataset
176 std::string folder_path = datasets_root_path_ + "/testPK/data/";
177 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
178 EXPECT_NE(ds, nullptr);
179
180 // Create a Repeat operation on ds
181 int32_t repeat_num = 3;
182 ds = ds->Repeat(repeat_num);
183 EXPECT_NE(ds, nullptr);
184
185 // Create auto contrast object
186 std::shared_ptr<TensorTransform> auto_contrast(new vision::AutoContrast(10, {10, 20}));
187 // Note: No need to check for output after calling API class constructor
188
189 // Create a Map operation on ds
190 ds = ds->Map({auto_contrast});
191 EXPECT_NE(ds, nullptr);
192
193 // Create a Batch operation on ds
194 int32_t batch_size = 1;
195 ds = ds->Batch(batch_size);
196 EXPECT_NE(ds, nullptr);
197
198 // Create an iterator over the result of the above dataset
199 // This will trigger the creation of the Execution Tree and launch it.
200 std::shared_ptr<Iterator> iter = ds->CreateIterator();
201 EXPECT_NE(iter, nullptr);
202
203 // Iterate the dataset and get each row
204 std::unordered_map<std::string, mindspore::MSTensor> row;
205 ASSERT_OK(iter->GetNextRow(&row));
206
207 uint64_t i = 0;
208 while (row.size() != 0) {
209 i++;
210 auto image = row["image"];
211 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
212 ASSERT_OK(iter->GetNextRow(&row));
213 }
214
215 EXPECT_EQ(i, 15);
216
217 // Manually terminate the pipeline
218 iter->Stop();
219 }
220
TEST_F(MindDataTestPipeline,TestCenterCrop)221 TEST_F(MindDataTestPipeline, TestCenterCrop) {
222 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCenterCrop with single integer input.";
223
224 // Create an ImageFolder Dataset
225 std::string folder_path = datasets_root_path_ + "/testPK/data/";
226 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
227 EXPECT_NE(ds, nullptr);
228
229 // Create a Repeat operation on ds
230 int32_t repeat_num = 3;
231 ds = ds->Repeat(repeat_num);
232 EXPECT_NE(ds, nullptr);
233
234 // Create centre crop object with square crop
235 std::shared_ptr<TensorTransform> centre_out1(new vision::CenterCrop({30}));
236 // Note: No need to check for output after calling API class constructor
237
238 // Create a Map operation on ds
239 ds = ds->Map({centre_out1});
240 EXPECT_NE(ds, nullptr);
241
242 // Create a Batch operation on ds
243 int32_t batch_size = 1;
244 ds = ds->Batch(batch_size);
245 EXPECT_NE(ds, nullptr);
246
247 // Create an iterator over the result of the above dataset
248 // This will trigger the creation of the Execution Tree and launch it.
249 std::shared_ptr<Iterator> iter = ds->CreateIterator();
250 EXPECT_NE(iter, nullptr);
251
252 // Iterate the dataset and get each row
253 std::unordered_map<std::string, mindspore::MSTensor> row;
254 ASSERT_OK(iter->GetNextRow(&row));
255
256 uint64_t i = 0;
257 while (row.size() != 0) {
258 i++;
259 auto image = row["image"];
260 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
261 ASSERT_OK(iter->GetNextRow(&row));
262 }
263
264 EXPECT_EQ(i, 15);
265
266 // Manually terminate the pipeline
267 iter->Stop();
268 }
269
TEST_F(MindDataTestPipeline,TestCropSuccess)270 TEST_F(MindDataTestPipeline, TestCropSuccess) {
271 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCropSuccess.";
272
273 // Create an ImageFolder Dataset
274 std::string folder_path = datasets_root_path_ + "/testPK/data/";
275 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
276 EXPECT_NE(ds, nullptr);
277
278 // Create a crop object
279 int height = 20;
280 int width = 25;
281 std::shared_ptr<TensorTransform> crop(new vision::Crop({0, 0}, {height, width}));
282 // Note: No need to check for output after calling API class constructor
283
284 // Create a Map operation on ds
285 ds = ds->Map({crop});
286 EXPECT_NE(ds, nullptr);
287
288 // Create a Batch operation on ds
289 int32_t batch_size = 1;
290 ds = ds->Batch(batch_size);
291 EXPECT_NE(ds, nullptr);
292
293 // Create an iterator over the result of the above dataset
294 // This will trigger the creation of the Execution Tree and launch it.
295 std::shared_ptr<Iterator> iter = ds->CreateIterator();
296 EXPECT_NE(iter, nullptr);
297
298 // Iterate the dataset and get each row
299 std::unordered_map<std::string, mindspore::MSTensor> row;
300 ASSERT_OK(iter->GetNextRow(&row));
301
302 uint64_t i = 0;
303 while (row.size() != 0) {
304 i++;
305 auto image = row["image"];
306 EXPECT_EQ(image.Shape()[1], height);
307 EXPECT_EQ(image.Shape()[2], width);
308 ASSERT_OK(iter->GetNextRow(&row));
309 }
310
311 EXPECT_EQ(i, 5);
312
313 // Manually terminate the pipeline
314 iter->Stop();
315 }
316
TEST_F(MindDataTestPipeline,TestCropParamCheck)317 TEST_F(MindDataTestPipeline, TestCropParamCheck) {
318 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCropParamCheck with invalid parameters.";
319
320 // Create an ImageFolder Dataset
321 std::string folder_path = datasets_root_path_ + "/testPK/data/";
322 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 5));
323 EXPECT_NE(ds, nullptr);
324
325 // Case 1: Value of coordinates is negative
326 // Create objects for the tensor ops
327 std::shared_ptr<TensorTransform> crop1(new vision::Crop({-1, -1}, {20}));
328 auto ds1 = ds->Map({crop1});
329 EXPECT_NE(ds1, nullptr);
330 // Create an iterator over the result of the above dataset
331 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
332 // Expect failure: invalid coordinates for Crop
333 EXPECT_EQ(iter1, nullptr);
334
335 // Case 2: Size of coordinates is not 2
336 // Create objects for the tensor ops
337 std::shared_ptr<TensorTransform> crop2(new vision::Crop({5}, {10}));
338 auto ds2 = ds->Map({crop2});
339 EXPECT_NE(ds2, nullptr);
340 // Create an iterator over the result of the above dataset
341 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
342 // Expect failure: invalid coordinates for Crop
343 EXPECT_EQ(iter2, nullptr);
344
345 // Case 3: Value of size is negative
346 // Create objects for the tensor ops
347 std::shared_ptr<TensorTransform> crop3(new vision::Crop({0, 0}, {-10, -5}));
348 auto ds3 = ds->Map({crop3});
349 EXPECT_NE(ds3, nullptr);
350 // Create an iterator over the result of the above dataset
351 std::shared_ptr<Iterator> iter3 = ds3->CreateIterator();
352 // Expect failure: invalid size for Crop
353 EXPECT_EQ(iter3, nullptr);
354
355 // Case 4: Size is neither a single number nor a vector of size 2
356 // Create objects for the tensor ops
357 std::shared_ptr<TensorTransform> crop4(new vision::Crop({0, 0}, {10, 10, 10}));
358 auto ds4 = ds->Map({crop4});
359 EXPECT_NE(ds4, nullptr);
360 // Create an iterator over the result of the above dataset
361 std::shared_ptr<Iterator> iter4 = ds4->CreateIterator();
362 // Expect failure: invalid size for Crop
363 EXPECT_EQ(iter4, nullptr);
364 }
365
TEST_F(MindDataTestPipeline,TestCutMixBatchSuccess1)366 TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) {
367 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchSuccess1.";
368 // Testing CutMixBatch on a batch of CHW images
369
370 // Create a Cifar10 Dataset
371 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
372 int number_of_classes = 10;
373 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
374 EXPECT_NE(ds, nullptr);
375
376 // Create objects for the tensor ops
377 std::shared_ptr<TensorTransform> hwc_to_chw = std::make_shared<vision::HWC2CHW>();
378 // Note: No need to check for output after calling API class constructor
379
380 // Create a Map operation on ds
381 ds = ds->Map({hwc_to_chw}, {"image"});
382 EXPECT_NE(ds, nullptr);
383
384 // Create a Batch operation on ds
385 int32_t batch_size = 5;
386 ds = ds->Batch(batch_size);
387 EXPECT_NE(ds, nullptr);
388
389 // Create objects for the tensor ops
390 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(number_of_classes);
391 // Note: No need to check for output after calling API class constructor
392
393 // Create a Map operation on ds
394 ds = ds->Map({one_hot_op}, {"label"});
395 EXPECT_NE(ds, nullptr);
396
397 std::shared_ptr<TensorTransform> cutmix_batch_op =
398 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNCHW, 1.0, 1.0);
399 // Note: No need to check for output after calling API class constructor
400
401 // Create a Map operation on ds
402 ds = ds->Map({cutmix_batch_op}, {"image", "label"});
403 EXPECT_NE(ds, nullptr);
404
405 // Create an iterator over the result of the above dataset
406 // This will trigger the creation of the Execution Tree and launch it.
407 std::shared_ptr<Iterator> iter = ds->CreateIterator();
408 EXPECT_NE(iter, nullptr);
409
410 // Iterate the dataset and get each row
411 std::unordered_map<std::string, mindspore::MSTensor> row;
412 ASSERT_OK(iter->GetNextRow(&row));
413
414 uint64_t i = 0;
415 while (row.size() != 0) {
416 i++;
417 auto image = row["image"];
418 auto label = row["label"];
419 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
420 MS_LOG(INFO) << "Label shape: " << label.Shape();
421 EXPECT_EQ(image.Shape().size() == 4 && batch_size == image.Shape()[0] && 3 == image.Shape()[1] &&
422 32 == image.Shape()[2] && 32 == image.Shape()[3],
423 true);
424 EXPECT_EQ(label.Shape().size() == 2 && batch_size == label.Shape()[0] && number_of_classes == label.Shape()[1],
425 true);
426 ASSERT_OK(iter->GetNextRow(&row));
427 }
428
429 EXPECT_EQ(i, 2);
430
431 // Manually terminate the pipeline
432 iter->Stop();
433 }
434
TEST_F(MindDataTestPipeline,TestCutMixBatchSuccess2)435 TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess2) {
436 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchSuccess2.";
437 // Calling CutMixBatch on a batch of HWC images with default values of alpha and prob
438
439 // Create a Cifar10 Dataset
440 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
441 int number_of_classes = 10;
442 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
443 EXPECT_NE(ds, nullptr);
444
445 // Create a Batch operation on ds
446 int32_t batch_size = 5;
447 ds = ds->Batch(batch_size);
448 EXPECT_NE(ds, nullptr);
449
450 // Create objects for the tensor ops
451 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(number_of_classes);
452 // Note: No need to check for output after calling API class constructor
453
454 // Create a Map operation on ds
455 ds = ds->Map({one_hot_op}, {"label"});
456 EXPECT_NE(ds, nullptr);
457
458 std::shared_ptr<TensorTransform> cutmix_batch_op =
459 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNHWC);
460 // Note: No need to check for output after calling API class constructor
461
462 // Create a Map operation on ds
463 ds = ds->Map({cutmix_batch_op}, {"image", "label"});
464 EXPECT_NE(ds, nullptr);
465
466 // Create an iterator over the result of the above dataset
467 // This will trigger the creation of the Execution Tree and launch it.
468 std::shared_ptr<Iterator> iter = ds->CreateIterator();
469 EXPECT_NE(iter, nullptr);
470
471 // Iterate the dataset and get each row
472 std::unordered_map<std::string, mindspore::MSTensor> row;
473 ASSERT_OK(iter->GetNextRow(&row));
474
475 uint64_t i = 0;
476 while (row.size() != 0) {
477 i++;
478 auto image = row["image"];
479 auto label = row["label"];
480 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
481 MS_LOG(INFO) << "Label shape: " << label.Shape();
482 EXPECT_EQ(image.Shape().size() == 4 && batch_size == image.Shape()[0] && 32 == image.Shape()[1] &&
483 32 == image.Shape()[2] && 3 == image.Shape()[3],
484 true);
485 EXPECT_EQ(label.Shape().size() == 2 && batch_size == label.Shape()[0] && number_of_classes == label.Shape()[1],
486 true);
487
488 ASSERT_OK(iter->GetNextRow(&row));
489 }
490
491 EXPECT_EQ(i, 2);
492
493 // Manually terminate the pipeline
494 iter->Stop();
495 }
496
TEST_F(MindDataTestPipeline,TestCutMixBatchFail1)497 TEST_F(MindDataTestPipeline, TestCutMixBatchFail1) {
498 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail1 with invalid negative alpha parameter.";
499
500 // Create a Cifar10 Dataset
501 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
502 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
503 EXPECT_NE(ds, nullptr);
504
505 // Create a Batch operation on ds
506 int32_t batch_size = 5;
507 ds = ds->Batch(batch_size);
508 EXPECT_NE(ds, nullptr);
509
510 // Create objects for the tensor ops
511 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
512 // Note: No need to check for output after calling API class constructor
513
514 // Create a Map operation on ds
515 ds = ds->Map({one_hot_op}, {"label"});
516 EXPECT_NE(ds, nullptr);
517
518 // Create CutMixBatch operation with invalid input, alpha<0
519 std::shared_ptr<TensorTransform> cutmix_batch_op =
520 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNHWC, -1, 0.5);
521 // Note: No need to check for output after calling API class constructor
522
523 // Create a Map operation on ds
524 ds = ds->Map({cutmix_batch_op});
525 EXPECT_NE(ds, nullptr);
526
527 std::shared_ptr<Iterator> iter = ds->CreateIterator();
528 // Expect failure: Invalid CutMixBatch input
529 EXPECT_EQ(iter, nullptr);
530 }
531
TEST_F(MindDataTestPipeline,TestCutMixBatchFail2)532 TEST_F(MindDataTestPipeline, TestCutMixBatchFail2) {
533 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail2 with invalid negative prob parameter.";
534
535 // Create a Cifar10 Dataset
536 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
537 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
538 EXPECT_NE(ds, nullptr);
539
540 // Create a Batch operation on ds
541 int32_t batch_size = 5;
542 ds = ds->Batch(batch_size);
543 EXPECT_NE(ds, nullptr);
544
545 // Create objects for the tensor ops
546 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
547 // Note: No need to check for output after calling API class constructor
548
549 // Create a Map operation on ds
550 ds = ds->Map({one_hot_op}, {"label"});
551 EXPECT_NE(ds, nullptr);
552
553 // Create CutMixBatch operation with invalid input, prob<0
554 std::shared_ptr<TensorTransform> cutmix_batch_op =
555 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNHWC, 1, -0.5);
556 // Note: No need to check for output after calling API class constructor
557
558 // Create a Map operation on ds
559 ds = ds->Map({cutmix_batch_op});
560 EXPECT_NE(ds, nullptr);
561
562 std::shared_ptr<Iterator> iter = ds->CreateIterator();
563 // Expect failure: Invalid CutMixBatch input
564 EXPECT_EQ(iter, nullptr);
565 }
566
TEST_F(MindDataTestPipeline,TestCutMixBatchFail3)567 TEST_F(MindDataTestPipeline, TestCutMixBatchFail3) {
568 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail3 with invalid zero alpha parameter.";
569
570 // Create a Cifar10 Dataset
571 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
572 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
573 EXPECT_NE(ds, nullptr);
574
575 // Create a Batch operation on ds
576 int32_t batch_size = 5;
577 ds = ds->Batch(batch_size);
578 EXPECT_NE(ds, nullptr);
579
580 // Create objects for the tensor ops
581 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
582 // Note: No need to check for output after calling API class constructor
583
584 // Create a Map operation on ds
585 ds = ds->Map({one_hot_op}, {"label"});
586 EXPECT_NE(ds, nullptr);
587
588 // Create CutMixBatch operation with invalid input, alpha=0 (boundary case)
589 std::shared_ptr<TensorTransform> cutmix_batch_op =
590 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNHWC, 0.0, 0.5);
591 // Note: No need to check for output after calling API class constructor
592
593 // Create a Map operation on ds
594 ds = ds->Map({cutmix_batch_op});
595 EXPECT_NE(ds, nullptr);
596
597 std::shared_ptr<Iterator> iter = ds->CreateIterator();
598 // Expect failure: Invalid CutMixBatch input
599 EXPECT_EQ(iter, nullptr);
600 }
601
TEST_F(MindDataTestPipeline,TestCutMixBatchFail4)602 TEST_F(MindDataTestPipeline, TestCutMixBatchFail4) {
603 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter.";
604
605 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
606 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
607 EXPECT_NE(ds, nullptr);
608
609 // Create a Batch operation on ds
610 int32_t batch_size = 10;
611 ds = ds->Batch(batch_size);
612 EXPECT_NE(ds, nullptr);
613
614 // Create objects for the tensor ops
615 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
616 // Note: No need to check for output after calling API class constructor
617
618 // Create a Map operation on ds
619 ds = ds->Map({one_hot_op}, {"label"});
620 EXPECT_NE(ds, nullptr);
621
622 // Create CutMixBatch operation with invalid input, prob>1
623 std::shared_ptr<TensorTransform> cutmix_batch_op =
624 std::make_shared<vision::CutMixBatch>(mindspore::dataset::ImageBatchFormat::kNHWC, 1, 1.5);
625 // Note: No need to check for output after calling API class constructor
626
627 // Create a Map operation on ds
628 ds = ds->Map({cutmix_batch_op});
629 EXPECT_NE(ds, nullptr);
630
631 std::shared_ptr<Iterator> iter = ds->CreateIterator();
632 // Expect failure: Invalid CutMixBatch input
633 EXPECT_EQ(iter, nullptr);
634 }
635
TEST_F(MindDataTestPipeline,TestCutOut)636 TEST_F(MindDataTestPipeline, TestCutOut) {
637 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutOut.";
638
639 // Create an ImageFolder Dataset
640 std::string folder_path = datasets_root_path_ + "/testPK/data/";
641 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
642 EXPECT_NE(ds, nullptr);
643
644 // Create a Repeat operation on ds
645 int32_t repeat_num = 2;
646 ds = ds->Repeat(repeat_num);
647 EXPECT_NE(ds, nullptr);
648
649 // Create objects for the tensor ops
650 std::shared_ptr<TensorTransform> cut_out1 = std::make_shared<vision::CutOut>(30, 5);
651 std::shared_ptr<TensorTransform> cut_out2 = std::make_shared<vision::CutOut>(30);
652 // Note: No need to check for output after calling API class constructor
653
654 // Create a Map operation on ds
655 ds = ds->Map({cut_out1, cut_out2});
656 EXPECT_NE(ds, nullptr);
657
658 // Create a Batch operation on ds
659 int32_t batch_size = 1;
660 ds = ds->Batch(batch_size);
661 EXPECT_NE(ds, nullptr);
662
663 // Create an iterator over the result of the above dataset
664 // This will trigger the creation of the Execution Tree and launch it.
665 std::shared_ptr<Iterator> iter = ds->CreateIterator();
666 EXPECT_NE(iter, nullptr);
667
668 // Iterate the dataset and get each row
669 std::unordered_map<std::string, mindspore::MSTensor> row;
670 ASSERT_OK(iter->GetNextRow(&row));
671
672 uint64_t i = 0;
673 while (row.size() != 0) {
674 i++;
675 auto image = row["image"];
676 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
677 ASSERT_OK(iter->GetNextRow(&row));
678 }
679
680 EXPECT_EQ(i, 20);
681
682 // Manually terminate the pipeline
683 iter->Stop();
684 }
685
TEST_F(MindDataTestPipeline,TestDecode)686 TEST_F(MindDataTestPipeline, TestDecode) {
687 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDecode.";
688
689 // Create an ImageFolder Dataset
690 std::string folder_path = datasets_root_path_ + "/testPK/data/";
691 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, std::make_shared<RandomSampler>(false, 10));
692 EXPECT_NE(ds, nullptr);
693
694 // Create a Repeat operation on ds
695 int32_t repeat_num = 2;
696 ds = ds->Repeat(repeat_num);
697 EXPECT_NE(ds, nullptr);
698
699 // Create Decode object
700 vision::Decode decode = vision::Decode(true);
701 // Note: No need to check for output after calling API class constructor
702
703 // Create a Map operation on ds
704 ds = ds->Map({decode});
705 EXPECT_NE(ds, nullptr);
706
707 // Create a Batch operation on ds
708 int32_t batch_size = 1;
709 ds = ds->Batch(batch_size);
710 EXPECT_NE(ds, nullptr);
711
712 // Create an iterator over the result of the above dataset
713 // This will trigger the creation of the Execution Tree and launch it.
714 std::shared_ptr<Iterator> iter = ds->CreateIterator();
715 EXPECT_NE(iter, nullptr);
716
717 // Iterate the dataset and get each row
718 std::unordered_map<std::string, mindspore::MSTensor> row;
719 ASSERT_OK(iter->GetNextRow(&row));
720
721 uint64_t i = 0;
722 while (row.size() != 0) {
723 i++;
724 auto image = row["image"];
725 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
726 ASSERT_OK(iter->GetNextRow(&row));
727 }
728 EXPECT_EQ(i, 20);
729
730 // Manually terminate the pipeline
731 iter->Stop();
732 }
733
TEST_F(MindDataTestPipeline,TestHwcToChw)734 TEST_F(MindDataTestPipeline, TestHwcToChw) {
735 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestHwcToChw.";
736
737 // Create an ImageFolder Dataset
738 std::string folder_path = datasets_root_path_ + "/testPK/data/";
739 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
740 EXPECT_NE(ds, nullptr);
741
742 // Create a Repeat operation on ds
743 int32_t repeat_num = 2;
744 ds = ds->Repeat(repeat_num);
745 EXPECT_NE(ds, nullptr);
746
747 // Create objects for the tensor ops
748 std::shared_ptr<TensorTransform> channel_swap = std::make_shared<vision::HWC2CHW>();
749 // Note: No need to check for output after calling API class constructor
750
751 // Create a Map operation on ds
752 ds = ds->Map({channel_swap});
753 EXPECT_NE(ds, nullptr);
754
755 // Create a Batch operation on ds
756 int32_t batch_size = 1;
757 ds = ds->Batch(batch_size);
758 EXPECT_NE(ds, nullptr);
759
760 // Create an iterator over the result of the above dataset
761 // This will trigger the creation of the Execution Tree and launch it.
762 std::shared_ptr<Iterator> iter = ds->CreateIterator();
763 EXPECT_NE(iter, nullptr);
764
765 // Iterate the dataset and get each row
766 std::unordered_map<std::string, mindspore::MSTensor> row;
767 ASSERT_OK(iter->GetNextRow(&row));
768
769 uint64_t i = 0;
770 while (row.size() != 0) {
771 i++;
772 auto image = row["image"];
773 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
774 // Check if the image is in NCHW
775 EXPECT_EQ(
776 batch_size == image.Shape()[0] && 3 == image.Shape()[1] && 2268 == image.Shape()[2] && 4032 == image.Shape()[3],
777 true);
778 ASSERT_OK(iter->GetNextRow(&row));
779 }
780 EXPECT_EQ(i, 20);
781
782 // Manually terminate the pipeline
783 iter->Stop();
784 }
785
TEST_F(MindDataTestPipeline,TestInvert)786 TEST_F(MindDataTestPipeline, TestInvert) {
787 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestInvert.";
788
789 // Create an ImageFolder Dataset
790 std::string folder_path = datasets_root_path_ + "/testPK/data/";
791 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 20));
792 EXPECT_NE(ds, nullptr);
793
794 // Create objects for the tensor ops
795 std::shared_ptr<TensorTransform> invert_op = std::make_shared<vision::Invert>();
796 // Note: No need to check for output after calling API class constructor
797
798 // Create a Map operation on ds
799 ds = ds->Map({invert_op});
800 EXPECT_NE(ds, nullptr);
801
802 // Create an iterator over the result of the above dataset
803 // This will trigger the creation of the Execution Tree and launch it.
804 std::shared_ptr<Iterator> iter = ds->CreateIterator();
805 EXPECT_NE(iter, nullptr);
806
807 // Iterate the dataset and get each row
808 std::unordered_map<std::string, mindspore::MSTensor> row;
809 ASSERT_OK(iter->GetNextRow(&row));
810
811 uint64_t i = 0;
812 while (row.size() != 0) {
813 i++;
814 auto image = row["image"];
815 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
816 ASSERT_OK(iter->GetNextRow(&row));
817 }
818 EXPECT_EQ(i, 20);
819
820 // Manually terminate the pipeline
821 iter->Stop();
822 }
823
TEST_F(MindDataTestPipeline,TestMixUpBatchFail1)824 TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) {
825 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMixUpBatchFail1 with negative alpha parameter.";
826
827 // Create a Cifar10 Dataset
828 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
829 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
830 EXPECT_NE(ds, nullptr);
831
832 // Create a Batch operation on ds
833 int32_t batch_size = 5;
834 ds = ds->Batch(batch_size);
835 EXPECT_NE(ds, nullptr);
836
837 // Create objects for the tensor ops
838 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
839 // Note: No need to check for output after calling API class constructor
840
841 // Create a Map operation on ds
842 ds = ds->Map({one_hot_op}, {"label"});
843 EXPECT_NE(ds, nullptr);
844
845 // Create MixUpBatch operation with invalid input, alpha<0
846 std::shared_ptr<TensorTransform> mixup_batch_op = std::make_shared<vision::MixUpBatch>(-1);
847 // Note: No need to check for output after calling API class constructor
848
849 // Create a Map operation on ds
850 ds = ds->Map({mixup_batch_op});
851 EXPECT_NE(ds, nullptr);
852
853 std::shared_ptr<Iterator> iter = ds->CreateIterator();
854 // Expect failure: Invalid MixUpBatch input
855 EXPECT_EQ(iter, nullptr);
856 }
857
TEST_F(MindDataTestPipeline,TestMixUpBatchFail2)858 TEST_F(MindDataTestPipeline, TestMixUpBatchFail2) {
859 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMixUpBatchFail2 with zero alpha parameter.";
860
861 // Create a Cifar10 Dataset
862 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
863 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
864 EXPECT_NE(ds, nullptr);
865
866 // Create a Batch operation on ds
867 int32_t batch_size = 5;
868 ds = ds->Batch(batch_size);
869 EXPECT_NE(ds, nullptr);
870
871 // Create objects for the tensor ops
872 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
873 // Note: No need to check for output after calling API class constructor
874
875 // Create a Map operation on ds
876 ds = ds->Map({one_hot_op}, {"label"});
877 EXPECT_NE(ds, nullptr);
878
879 // Create MixUpBatch operation with invalid input, alpha<0 (boundary case)
880 std::shared_ptr<TensorTransform> mixup_batch_op = std::make_shared<vision::MixUpBatch>(0.0);
881 // Note: No need to check for output after calling API class constructor
882
883 // Create a Map operation on ds
884 ds = ds->Map({mixup_batch_op});
885 EXPECT_NE(ds, nullptr);
886
887 std::shared_ptr<Iterator> iter = ds->CreateIterator();
888 // Expect failure: Invalid MixUpBatch input
889 EXPECT_EQ(iter, nullptr);
890 }
891
TEST_F(MindDataTestPipeline,TestMixUpBatchSuccess1)892 TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) {
893 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMixUpBatchSuccess1 with explicit alpha parameter.";
894
895 // Create a Cifar10 Dataset
896 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
897 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
898 EXPECT_NE(ds, nullptr);
899
900 // Create a Batch operation on ds
901 int32_t batch_size = 5;
902 ds = ds->Batch(batch_size);
903 EXPECT_NE(ds, nullptr);
904
905 // Create objects for the tensor ops
906 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
907 // Note: No need to check for output after calling API class constructor
908
909 // Create a Map operation on ds
910 ds = ds->Map({one_hot_op}, {"label"});
911 EXPECT_NE(ds, nullptr);
912
913 std::shared_ptr<TensorTransform> mixup_batch_op = std::make_shared<vision::MixUpBatch>(2.0);
914 // Note: No need to check for output after calling API class constructor
915
916 // Create a Map operation on ds
917 ds = ds->Map({mixup_batch_op}, {"image", "label"});
918 EXPECT_NE(ds, nullptr);
919
920 // Create an iterator over the result of the above dataset
921 // This will trigger the creation of the Execution Tree and launch it.
922 std::shared_ptr<Iterator> iter = ds->CreateIterator();
923 EXPECT_NE(iter, nullptr);
924
925 // Iterate the dataset and get each row
926 std::unordered_map<std::string, mindspore::MSTensor> row;
927 ASSERT_OK(iter->GetNextRow(&row));
928
929 uint64_t i = 0;
930 while (row.size() != 0) {
931 i++;
932 auto image = row["image"];
933 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
934 ASSERT_OK(iter->GetNextRow(&row));
935 }
936
937 EXPECT_EQ(i, 2);
938
939 // Manually terminate the pipeline
940 iter->Stop();
941 }
942
TEST_F(MindDataTestPipeline,TestMixUpBatchSuccess2)943 TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) {
944 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMixUpBatchSuccess1 with default alpha parameter.";
945
946 // Create a Cifar10 Dataset
947 std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
948 std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", std::make_shared<RandomSampler>(false, 10));
949 EXPECT_NE(ds, nullptr);
950
951 // Create a Batch operation on ds
952 int32_t batch_size = 5;
953 ds = ds->Batch(batch_size);
954 EXPECT_NE(ds, nullptr);
955
956 // Create objects for the tensor ops
957 std::shared_ptr<TensorTransform> one_hot_op = std::make_shared<transforms::OneHot>(10);
958 // Note: No need to check for output after calling API class constructor
959
960 // Create a Map operation on ds
961 ds = ds->Map({one_hot_op}, {"label"});
962 EXPECT_NE(ds, nullptr);
963
964 std::shared_ptr<TensorTransform> mixup_batch_op = std::make_shared<vision::MixUpBatch>();
965 // Note: No need to check for output after calling API class constructor
966
967 // Create a Map operation on ds
968 ds = ds->Map({mixup_batch_op}, {"image", "label"});
969 EXPECT_NE(ds, nullptr);
970
971 // Create an iterator over the result of the above dataset
972 // This will trigger the creation of the Execution Tree and launch it.
973 std::shared_ptr<Iterator> iter = ds->CreateIterator();
974 EXPECT_NE(iter, nullptr);
975
976 // Iterate the dataset and get each row
977 std::unordered_map<std::string, mindspore::MSTensor> row;
978 ASSERT_OK(iter->GetNextRow(&row));
979
980 uint64_t i = 0;
981 while (row.size() != 0) {
982 i++;
983 auto image = row["image"];
984 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
985 ASSERT_OK(iter->GetNextRow(&row));
986 }
987
988 EXPECT_EQ(i, 2);
989
990 // Manually terminate the pipeline
991 iter->Stop();
992 }
993
TEST_F(MindDataTestPipeline,TestNormalize)994 TEST_F(MindDataTestPipeline, TestNormalize) {
995 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalize.";
996
997 // Create an ImageFolder Dataset
998 std::string folder_path = datasets_root_path_ + "/testPK/data/";
999 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
1000 EXPECT_NE(ds, nullptr);
1001
1002 // Create a Repeat operation on ds
1003 int32_t repeat_num = 2;
1004 ds = ds->Repeat(repeat_num);
1005 EXPECT_NE(ds, nullptr);
1006
1007 // Create objects for the tensor ops
1008 std::shared_ptr<TensorTransform> normalize(new vision::Normalize({121.0, 115.0, 0.0}, {70.0, 68.0, 71.0}));
1009 // Note: No need to check for output after calling API class constructor
1010
1011 // Create a Map operation on ds
1012 ds = ds->Map({normalize});
1013 EXPECT_NE(ds, nullptr);
1014
1015 // Create a Batch operation on ds
1016 int32_t batch_size = 1;
1017 ds = ds->Batch(batch_size);
1018 EXPECT_NE(ds, nullptr);
1019
1020 // Create an iterator over the result of the above dataset
1021 // This will trigger the creation of the Execution Tree and launch it.
1022 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1023 EXPECT_NE(iter, nullptr);
1024
1025 // Iterate the dataset and get each row
1026 std::unordered_map<std::string, mindspore::MSTensor> row;
1027 ASSERT_OK(iter->GetNextRow(&row));
1028
1029 uint64_t i = 0;
1030 while (row.size() != 0) {
1031 i++;
1032 auto image = row["image"];
1033 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1034 ASSERT_OK(iter->GetNextRow(&row));
1035 }
1036
1037 EXPECT_EQ(i, 20);
1038
1039 // Manually terminate the pipeline
1040 iter->Stop();
1041 }
1042
TEST_F(MindDataTestPipeline,TestNormalizePad)1043 TEST_F(MindDataTestPipeline, TestNormalizePad) {
1044 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizePad.";
1045
1046 // Create an ImageFolder Dataset
1047 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1048 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
1049 EXPECT_NE(ds, nullptr);
1050
1051 // Create a Repeat operation on ds
1052 int32_t repeat_num = 2;
1053 ds = ds->Repeat(repeat_num);
1054 EXPECT_NE(ds, nullptr);
1055
1056 // Create objects for the tensor ops
1057 std::shared_ptr<TensorTransform> normalizepad(
1058 new vision::NormalizePad({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0}, "float32"));
1059 // Note: No need to check for output after calling API class constructor
1060
1061 // Create a Map operation on ds
1062 ds = ds->Map({normalizepad});
1063 EXPECT_NE(ds, nullptr);
1064
1065 // Create an iterator over the result of the above dataset
1066 // This will trigger the creation of the Execution Tree and launch it.
1067 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1068 EXPECT_NE(iter, nullptr);
1069
1070 // Iterate the dataset and get each row
1071 std::unordered_map<std::string, mindspore::MSTensor> row;
1072 ASSERT_OK(iter->GetNextRow(&row));
1073
1074 uint64_t i = 0;
1075 while (row.size() != 0) {
1076 i++;
1077 auto image = row["image"];
1078 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1079 EXPECT_EQ(image.Shape()[2], 4);
1080
1081 ASSERT_OK(iter->GetNextRow(&row));
1082 }
1083
1084 EXPECT_EQ(i, 20);
1085
1086 // Manually terminate the pipeline
1087 iter->Stop();
1088 }
1089
TEST_F(MindDataTestPipeline,TestPad)1090 TEST_F(MindDataTestPipeline, TestPad) {
1091 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestPad.";
1092
1093 // Create an ImageFolder Dataset
1094 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1095 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
1096 EXPECT_NE(ds, nullptr);
1097
1098 // Create a Repeat operation on ds
1099 int32_t repeat_num = 2;
1100 ds = ds->Repeat(repeat_num);
1101 EXPECT_NE(ds, nullptr);
1102
1103 // Create objects for the tensor ops
1104 std::shared_ptr<TensorTransform> pad_op1(new vision::Pad({1, 2, 3, 4}, {0}, BorderType::kSymmetric));
1105 std::shared_ptr<TensorTransform> pad_op2(new vision::Pad({1}, {1, 1, 1}, BorderType::kEdge));
1106 std::shared_ptr<TensorTransform> pad_op3(new vision::Pad({1, 4}));
1107 // Note: No need to check for output after calling API class constructor
1108
1109 // Create a Map operation on ds
1110 ds = ds->Map({pad_op1, pad_op2, pad_op3});
1111 EXPECT_NE(ds, nullptr);
1112
1113 // Create a Batch operation on ds
1114 int32_t batch_size = 1;
1115 ds = ds->Batch(batch_size);
1116 EXPECT_NE(ds, nullptr);
1117
1118 // Create an iterator over the result of the above dataset
1119 // This will trigger the creation of the Execution Tree and launch it.
1120 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1121 EXPECT_NE(iter, nullptr);
1122
1123 // Iterate the dataset and get each row
1124 std::unordered_map<std::string, mindspore::MSTensor> row;
1125 ASSERT_OK(iter->GetNextRow(&row));
1126
1127 uint64_t i = 0;
1128 while (row.size() != 0) {
1129 i++;
1130 auto image = row["image"];
1131 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1132 ASSERT_OK(iter->GetNextRow(&row));
1133 }
1134
1135 EXPECT_EQ(i, 20);
1136
1137 // Manually terminate the pipeline
1138 iter->Stop();
1139 }
1140
TEST_F(MindDataTestPipeline,TestConvertColorSuccess1)1141 TEST_F(MindDataTestPipeline, TestConvertColorSuccess1) {
1142 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConvertColorSuccess1.";
1143 // Create an ImageFolder Dataset
1144 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1145 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
1146 EXPECT_NE(ds, nullptr);
1147 // Create objects for the tensor ops
1148 std::shared_ptr<TensorTransform> resize_op(new vision::Resize({500, 1000}));
1149 std::shared_ptr<TensorTransform> convert(new mindspore::dataset::vision::ConvertColor(ConvertMode::COLOR_RGB2GRAY));
1150
1151 ds = ds->Map({resize_op, convert});
1152 EXPECT_NE(ds, nullptr);
1153
1154 // Create an iterator over the result of the above dataset
1155 // This will trigger the creation of the Execution Tree and launch it.
1156 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1157 EXPECT_NE(iter, nullptr);
1158
1159 // Iterate the dataset and get each row
1160 std::unordered_map<std::string, mindspore::MSTensor> row;
1161 ASSERT_OK(iter->GetNextRow(&row));
1162
1163 uint64_t i = 0;
1164 while (row.size() != 0) {
1165 i++;
1166 auto image = row["image"];
1167 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1168 EXPECT_EQ(image.Shape().size(), 2);
1169 ASSERT_OK(iter->GetNextRow(&row));
1170 }
1171
1172 EXPECT_EQ(i, 1);
1173
1174 // Manually terminate the pipeline
1175 iter->Stop();
1176 }
1177
TEST_F(MindDataTestPipeline,TestConvertColorSuccess2)1178 TEST_F(MindDataTestPipeline, TestConvertColorSuccess2) {
1179 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConvertColorSuccess2.";
1180 // Create an ImageFolder Dataset
1181 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1182 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
1183 EXPECT_NE(ds, nullptr);
1184 // Create objects for the tensor ops
1185 std::shared_ptr<TensorTransform> resize_op(new vision::Resize({500, 1000}));
1186 std::shared_ptr<TensorTransform> convert(new mindspore::dataset::vision::ConvertColor(ConvertMode::COLOR_RGB2BGR));
1187
1188 ds = ds->Map({resize_op, convert});
1189 EXPECT_NE(ds, nullptr);
1190
1191 // Create an iterator over the result of the above dataset
1192 // This will trigger the creation of the Execution Tree and launch it.
1193 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1194 EXPECT_NE(iter, nullptr);
1195
1196 // Iterate the dataset and get each row
1197 std::unordered_map<std::string, mindspore::MSTensor> row;
1198 ASSERT_OK(iter->GetNextRow(&row));
1199
1200 uint64_t i = 0;
1201 while (row.size() != 0) {
1202 i++;
1203 auto image = row["image"];
1204 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1205 EXPECT_EQ(image.Shape()[2], 3);
1206 ASSERT_OK(iter->GetNextRow(&row));
1207 }
1208
1209 EXPECT_EQ(i, 1);
1210
1211 // Manually terminate the pipeline
1212 iter->Stop();
1213 }
1214
TEST_F(MindDataTestPipeline,TestConvertColorSuccess3)1215 TEST_F(MindDataTestPipeline, TestConvertColorSuccess3) {
1216 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConvertColorSuccess3.";
1217 // Create an ImageFolder Dataset
1218 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1219 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
1220 EXPECT_NE(ds, nullptr);
1221 // Create objects for the tensor ops
1222 std::shared_ptr<TensorTransform> resize_op(new vision::Resize({500, 1000}));
1223 std::shared_ptr<TensorTransform> convert(new mindspore::dataset::vision::ConvertColor(ConvertMode::COLOR_RGB2RGBA));
1224
1225 ds = ds->Map({resize_op, convert});
1226 EXPECT_NE(ds, nullptr);
1227
1228 // Create an iterator over the result of the above dataset
1229 // This will trigger the creation of the Execution Tree and launch it.
1230 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1231 EXPECT_NE(iter, nullptr);
1232
1233 // Iterate the dataset and get each row
1234 std::unordered_map<std::string, mindspore::MSTensor> row;
1235 ASSERT_OK(iter->GetNextRow(&row));
1236
1237 uint64_t i = 0;
1238 while (row.size() != 0) {
1239 i++;
1240 auto image = row["image"];
1241 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
1242 EXPECT_EQ(image.Shape()[2], 4);
1243 ASSERT_OK(iter->GetNextRow(&row));
1244 }
1245
1246 EXPECT_EQ(i, 1);
1247
1248 // Manually terminate the pipeline
1249 iter->Stop();
1250 }
1251
TEST_F(MindDataTestPipeline,TestConvertColorFail)1252 TEST_F(MindDataTestPipeline, TestConvertColorFail) {
1253 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestConvertColorFail.";
1254 // Create an ImageFolder Dataset
1255 std::string folder_path = datasets_root_path_ + "/testPK/data/";
1256 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 1));
1257 EXPECT_NE(ds, nullptr);
1258
1259 ConvertMode error_convert_mode = static_cast<ConvertMode>(50);
1260
1261 // Create objects for the tensor ops
1262 std::shared_ptr<TensorTransform> resize_op(new vision::Resize({500, 1000}));
1263 std::shared_ptr<TensorTransform> convert(new mindspore::dataset::vision::ConvertColor(error_convert_mode));
1264
1265 ds = ds->Map({resize_op, convert});
1266 EXPECT_NE(ds, nullptr);
1267
1268 // Create an iterator over the result of the above dataset
1269 // This will trigger the creation of the Execution Tree and launch it.
1270 std::shared_ptr<Iterator> iter = ds->CreateIterator();
1271 EXPECT_EQ(iter, nullptr);
1272 }
1273