1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18 #include "minddata/dataset/core/global_context.h"
19
20 #include "ir/dtype/type_id.h"
21
22 using namespace mindspore::dataset;
23
24 class MindDataTestPipeline : public UT::DatasetOpTesting {
25 protected:
26 };
27
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic1)28 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) {
29 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic1.";
30
31 // Create a RandomDataset
32 std::shared_ptr<SchemaObj> schema = Schema();
33 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
34 ASSERT_OK(schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}));
35 std::shared_ptr<Dataset> ds = RandomData(50, schema);
36 EXPECT_NE(ds, nullptr);
37
38 ds = ds->SetNumWorkers(4);
39 EXPECT_NE(ds, nullptr);
40
41 // Create a Repeat operation on ds
42 ds = ds->Repeat(4);
43 EXPECT_NE(ds, nullptr);
44
45 // Create an iterator over the result of the above dataset
46 // This will trigger the creation of the Execution Tree and launch it.
47 std::shared_ptr<Iterator> iter = ds->CreateIterator();
48 EXPECT_NE(iter, nullptr);
49
50 // Iterate the dataset and get each row
51 std::unordered_map<std::string, mindspore::MSTensor> row;
52 ASSERT_OK(iter->GetNextRow(&row));
53
54 // Check if RandomData() read correct columns
55 uint64_t i = 0;
56 while (row.size() != 0) {
57 auto image = row["image"];
58 auto label = row["label"];
59 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
60 MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
61
62 ASSERT_OK(iter->GetNextRow(&row));
63 i++;
64 }
65
66 EXPECT_EQ(i, 200);
67
68 // Manually terminate the pipeline
69 iter->Stop();
70 }
71
TEST_F(MindDataTestPipeline,TestRandomDatasetBasicWithPipeline)72 TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) {
73 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasicWithPipeline.";
74
75 // Create two RandomDataset
76 std::shared_ptr<SchemaObj> schema = Schema();
77 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
78 ASSERT_OK(schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}));
79 std::shared_ptr<Dataset> ds1 = RandomData(50, schema);
80 std::shared_ptr<Dataset> ds2 = RandomData(50, schema);
81 EXPECT_NE(ds1, nullptr);
82 EXPECT_NE(ds2, nullptr);
83
84 // Create two Repeat operation on ds
85 int32_t repeat_num = 2;
86 ds1 = ds1->Repeat(repeat_num);
87 EXPECT_NE(ds1, nullptr);
88 repeat_num = 2;
89 ds2 = ds2->Repeat(repeat_num);
90 EXPECT_NE(ds2, nullptr);
91
92 // Create two Project operation on ds
93 std::vector<std::string> column_project = {"image", "label"};
94 ds1 = ds1->Project(column_project);
95 EXPECT_NE(ds1, nullptr);
96 ds2 = ds2->Project(column_project);
97 EXPECT_NE(ds2, nullptr);
98
99 // Create a Concat operation on the ds
100 ds1 = ds1->Concat({ds2});
101 EXPECT_NE(ds1, nullptr);
102
103 // Create an iterator over the result of the above dataset
104 // This will trigger the creation of the Execution Tree and launch it.
105 std::shared_ptr<Iterator> iter = ds1->CreateIterator();
106 EXPECT_NE(iter, nullptr);
107
108 // Iterate the dataset and get each row
109 std::unordered_map<std::string, mindspore::MSTensor> row;
110 ASSERT_OK(iter->GetNextRow(&row));
111
112 // Check if RandomData() read correct columns
113 uint64_t i = 0;
114 while (row.size() != 0) {
115 auto image = row["image"];
116 auto label = row["label"];
117 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
118 MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
119
120 ASSERT_OK(iter->GetNextRow(&row));
121 i++;
122 }
123
124 EXPECT_EQ(i, 200);
125
126 // Manually terminate the pipeline
127 iter->Stop();
128 }
129
TEST_F(MindDataTestPipeline,TestRandomDatasetGetters)130 TEST_F(MindDataTestPipeline, TestRandomDatasetGetters) {
131 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetGetters.";
132
133 // Create a RandomDataset
134 std::shared_ptr<SchemaObj> schema = Schema();
135 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
136 ASSERT_OK(schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}));
137 std::shared_ptr<Dataset> ds = RandomData(50, schema);
138 EXPECT_NE(ds, nullptr);
139
140 std::vector<std::string> column_names = {"image", "label"};
141 EXPECT_EQ(ds->GetDatasetSize(), 50);
142 EXPECT_EQ(ds->GetColumnNames(), column_names);
143 }
144
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic2)145 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic2) {
146 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic2.";
147
148 // Create a RandomDataset
149 std::shared_ptr<Dataset> ds = RandomData(10);
150 EXPECT_NE(ds, nullptr);
151
152 ds = ds->SetNumWorkers(1);
153 EXPECT_NE(ds, nullptr);
154
155 // Create a Repeat operation on ds
156 ds = ds->Repeat(2);
157 EXPECT_NE(ds, nullptr);
158
159 // Create an iterator over the result of the above dataset
160 // This will trigger the creation of the Execution Tree and launch it.
161 std::shared_ptr<Iterator> iter = ds->CreateIterator();
162 EXPECT_NE(iter, nullptr);
163
164 // Iterate the dataset and get each row
165 std::unordered_map<std::string, mindspore::MSTensor> row;
166 ASSERT_OK(iter->GetNextRow(&row));
167
168 // Check if RandomData() read correct columns
169 uint64_t i = 0;
170 while (row.size() != 0) {
171 // If no schema specified, RandomData will generate random columns
172 // So we don't check columns here
173 ASSERT_OK(iter->GetNextRow(&row));
174 i++;
175 }
176
177 EXPECT_EQ(i, 20);
178
179 // Manually terminate the pipeline
180 iter->Stop();
181 }
182
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic3)183 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic3) {
184 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic3.";
185
186 // Create a RandomDataset
187 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
188 GlobalContext::config_manager()->set_seed(246);
189
190 std::string SCHEMA_FILE = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
191 std::shared_ptr<SchemaObj> schema = Schema(SCHEMA_FILE);
192 std::shared_ptr<Dataset> ds = RandomData(0, schema);
193 EXPECT_NE(ds, nullptr);
194
195 // Create a Repeat operation on ds
196 ds = ds->Repeat(2);
197 EXPECT_NE(ds, nullptr);
198
199 // Create an iterator over the result of the above dataset
200 // This will trigger the creation of the Execution Tree and launch it.
201 std::shared_ptr<Iterator> iter = ds->CreateIterator();
202 EXPECT_NE(iter, nullptr);
203
204 // Iterate the dataset and get each row
205 std::unordered_map<std::string, mindspore::MSTensor> row;
206 ASSERT_OK(iter->GetNextRow(&row));
207
208 std::vector<int64_t> expect_num = {1};
209 std::vector<int64_t> expect_1d = {2};
210 std::vector<int64_t> expect_2d = {2, 2};
211 std::vector<int64_t> expect_3d = {2, 2, 2};
212
213 // Check if RandomData() read correct columns
214 uint64_t i = 0;
215 while (row.size() != 0) {
216 auto col_sint16 = row["col_sint16"];
217 auto col_sint32 = row["col_sint32"];
218 auto col_sint64 = row["col_sint64"];
219 auto col_float = row["col_float"];
220 auto col_1d = row["col_1d"];
221 auto col_2d = row["col_2d"];
222 auto col_3d = row["col_3d"];
223 auto col_binary = row["col_binary"];
224
225 // Validate shape
226 ASSERT_EQ(col_sint16.Shape(), expect_num);
227 ASSERT_EQ(col_sint32.Shape(), expect_num);
228 ASSERT_EQ(col_sint64.Shape(), expect_num);
229 ASSERT_EQ(col_float.Shape(), expect_num);
230 ASSERT_EQ(col_1d.Shape(), expect_1d);
231 ASSERT_EQ(col_2d.Shape(), expect_2d);
232 ASSERT_EQ(col_3d.Shape(), expect_3d);
233 ASSERT_EQ(col_binary.Shape(), expect_num);
234
235 // Validate Rank
236 ASSERT_EQ(col_sint16.Shape().size(), 1);
237 ASSERT_EQ(col_sint32.Shape().size(), 1);
238 ASSERT_EQ(col_sint64.Shape().size(), 1);
239 ASSERT_EQ(col_float.Shape().size(), 1);
240 ASSERT_EQ(col_1d.Shape().size(), 1);
241 ASSERT_EQ(col_2d.Shape().size(), 2);
242 ASSERT_EQ(col_3d.Shape().size(), 3);
243 ASSERT_EQ(col_binary.Shape().size(), 1);
244
245 // Validate type
246 ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16);
247 ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32);
248 ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64);
249 ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32);
250 ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64);
251 ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64);
252 ASSERT_EQ(col_3d.DataType(), mindspore::DataType::kNumberTypeInt64);
253 ASSERT_EQ(col_binary.DataType(), mindspore::DataType::kNumberTypeUInt8);
254
255 ASSERT_OK(iter->GetNextRow(&row));
256 i++;
257 }
258
259 EXPECT_EQ(i, 984);
260
261 // Manually terminate the pipeline
262 iter->Stop();
263 GlobalContext::config_manager()->set_seed(curr_seed);
264 }
265
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic4)266 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic4) {
267 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic4.";
268
269 // Create a RandomDataset
270 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
271 GlobalContext::config_manager()->set_seed(246);
272
273 std::string SCHEMA_FILE = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
274 std::shared_ptr<Dataset> ds = RandomData(0, SCHEMA_FILE);
275 EXPECT_NE(ds, nullptr);
276
277 // Create a Repeat operation on ds
278 ds = ds->Repeat(2);
279 EXPECT_NE(ds, nullptr);
280
281 // Create an iterator over the result of the above dataset
282 // This will trigger the creation of the Execution Tree and launch it.
283 std::shared_ptr<Iterator> iter = ds->CreateIterator();
284 EXPECT_NE(iter, nullptr);
285
286 // Iterate the dataset and get each row
287 std::unordered_map<std::string, mindspore::MSTensor> row;
288 ASSERT_OK(iter->GetNextRow(&row));
289
290 std::vector<int64_t> expect_num = {1};
291 std::vector<int64_t> expect_1d = {2};
292 std::vector<int64_t> expect_2d = {2, 2};
293 std::vector<int64_t> expect_3d = {2, 2, 2};
294
295 // Check if RandomData() read correct columns
296 uint64_t i = 0;
297 while (row.size() != 0) {
298 auto col_sint16 = row["col_sint16"];
299 auto col_sint32 = row["col_sint32"];
300 auto col_sint64 = row["col_sint64"];
301 auto col_float = row["col_float"];
302 auto col_1d = row["col_1d"];
303 auto col_2d = row["col_2d"];
304 auto col_3d = row["col_3d"];
305 auto col_binary = row["col_binary"];
306
307 // Validate shape
308 ASSERT_EQ(col_sint16.Shape(), expect_num);
309 ASSERT_EQ(col_sint32.Shape(), expect_num);
310 ASSERT_EQ(col_sint64.Shape(), expect_num);
311 ASSERT_EQ(col_float.Shape(), expect_num);
312 ASSERT_EQ(col_1d.Shape(), expect_1d);
313 ASSERT_EQ(col_2d.Shape(), expect_2d);
314 ASSERT_EQ(col_3d.Shape(), expect_3d);
315 ASSERT_EQ(col_binary.Shape(), expect_num);
316
317 // Validate Rank
318 ASSERT_EQ(col_sint16.Shape().size(), 1);
319 ASSERT_EQ(col_sint32.Shape().size(), 1);
320 ASSERT_EQ(col_sint64.Shape().size(), 1);
321 ASSERT_EQ(col_float.Shape().size(), 1);
322 ASSERT_EQ(col_1d.Shape().size(), 1);
323 ASSERT_EQ(col_2d.Shape().size(), 2);
324 ASSERT_EQ(col_3d.Shape().size(), 3);
325 ASSERT_EQ(col_binary.Shape().size(), 1);
326
327 // Validate type
328 ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16);
329 ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32);
330 ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64);
331 ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32);
332 ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64);
333 ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64);
334 ASSERT_EQ(col_3d.DataType(), mindspore::DataType::kNumberTypeInt64);
335 ASSERT_EQ(col_binary.DataType(), mindspore::DataType::kNumberTypeUInt8);
336
337 ASSERT_OK(iter->GetNextRow(&row));
338 i++;
339 }
340
341 EXPECT_EQ(i, 984);
342
343 // Manually terminate the pipeline
344 iter->Stop();
345 GlobalContext::config_manager()->set_seed(curr_seed);
346 }
347
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic5)348 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic5) {
349 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic5.";
350
351 // Create a RandomDataset
352 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
353 GlobalContext::config_manager()->set_seed(246);
354
355 std::string SCHEMA_FILE = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
356 std::shared_ptr<Dataset> ds = RandomData(0, SCHEMA_FILE, {"col_sint32", "col_sint64", "col_1d"});
357 EXPECT_NE(ds, nullptr);
358
359 // Create a Repeat operation on ds
360 ds = ds->Repeat(2);
361 EXPECT_NE(ds, nullptr);
362
363 // Create an iterator over the result of the above dataset
364 // This will trigger the creation of the Execution Tree and launch it.
365 std::shared_ptr<Iterator> iter = ds->CreateIterator();
366 EXPECT_NE(iter, nullptr);
367
368 // Iterate the dataset and get each row
369 std::unordered_map<std::string, mindspore::MSTensor> row;
370 ASSERT_OK(iter->GetNextRow(&row));
371
372 std::vector<int64_t> expect_num = {1};
373 std::vector<int64_t> expect_1d = {2};
374
375 // Check if RandomData() read correct columns
376 uint64_t i = 0;
377 while (row.size() != 0) {
378 EXPECT_EQ(row.size(), 3);
379
380 auto col_sint32 = row["col_sint32"];
381 auto col_sint64 = row["col_sint64"];
382 auto col_1d = row["col_1d"];
383
384 // Validate shape
385 ASSERT_EQ(col_sint32.Shape(), expect_num);
386 ASSERT_EQ(col_sint64.Shape(), expect_num);
387 ASSERT_EQ(col_1d.Shape(), expect_1d);
388
389 // Validate Rank
390 ASSERT_EQ(col_sint32.Shape().size(), 1);
391 ASSERT_EQ(col_sint64.Shape().size(), 1);
392 ASSERT_EQ(col_1d.Shape().size(), 1);
393
394 // Validate type
395 ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32);
396 ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64);
397 ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64);
398
399 ASSERT_OK(iter->GetNextRow(&row));
400 i++;
401 }
402
403 EXPECT_EQ(i, 984);
404
405 // Manually terminate the pipeline
406 iter->Stop();
407 GlobalContext::config_manager()->set_seed(curr_seed);
408 }
409
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic6)410 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic6) {
411 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic6.";
412
413 // Create a RandomDataset
414 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
415 GlobalContext::config_manager()->set_seed(246);
416
417 std::string SCHEMA_FILE = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
418 std::shared_ptr<Dataset> ds = RandomData(10, nullptr, {"col_sint32", "col_sint64", "col_1d"});
419 EXPECT_NE(ds, nullptr);
420
421 // Create an iterator over the result of the above dataset
422 // This will trigger the creation of the Execution Tree and launch it.
423 std::shared_ptr<Iterator> iter = ds->CreateIterator();
424 EXPECT_NE(iter, nullptr);
425
426 // Iterate the dataset and get each row
427 std::unordered_map<std::string, mindspore::MSTensor> row;
428 ASSERT_OK(iter->GetNextRow(&row));
429
430 // Check if RandomData() read correct columns
431 uint64_t i = 0;
432 while (row.size() != 0) {
433 ASSERT_OK(iter->GetNextRow(&row));
434 i++;
435 }
436
437 EXPECT_EQ(i, 10);
438
439 // Manually terminate the pipeline
440 iter->Stop();
441 GlobalContext::config_manager()->set_seed(curr_seed);
442 }
443
TEST_F(MindDataTestPipeline,TestRandomDatasetBasic7)444 TEST_F(MindDataTestPipeline, TestRandomDatasetBasic7) {
445 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetBasic7.";
446
447 // Create a RandomDataset
448 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
449 GlobalContext::config_manager()->set_seed(246);
450
451 std::string SCHEMA_FILE = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
452 std::shared_ptr<Dataset> ds = RandomData(10, "", {"col_sint32", "col_sint64", "col_1d"});
453 EXPECT_NE(ds, nullptr);
454
455 // Create an iterator over the result of the above dataset
456 // This will trigger the creation of the Execution Tree and launch it.
457 std::shared_ptr<Iterator> iter = ds->CreateIterator();
458 EXPECT_NE(iter, nullptr);
459
460 // Iterate the dataset and get each row
461 std::unordered_map<std::string, mindspore::MSTensor> row;
462 ASSERT_OK(iter->GetNextRow(&row));
463
464 // Check if RandomData() read correct columns
465 uint64_t i = 0;
466 while (row.size() != 0) {
467 ASSERT_OK(iter->GetNextRow(&row));
468 i++;
469 }
470
471 EXPECT_EQ(i, 10);
472
473 // Manually terminate the pipeline
474 iter->Stop();
475 GlobalContext::config_manager()->set_seed(curr_seed);
476 }
477
TEST_F(MindDataTestPipeline,TestRandomDatasetUInt8)478 TEST_F(MindDataTestPipeline, TestRandomDatasetUInt8) {
479 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetUInt8.";
480
481 // Create a RandomDataset with UInt8 numbers for given shape
482 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
483 GlobalContext::config_manager()->set_seed(963);
484 std::shared_ptr<SchemaObj> schema = Schema();
485 ASSERT_OK(schema->add_column("col1", mindspore::DataType::kNumberTypeUInt8, {4}));
486 std::shared_ptr<Dataset> ds = RandomData(3, schema);
487 EXPECT_NE(ds, nullptr);
488 ds = ds->SetNumWorkers(3);
489 EXPECT_NE(ds, nullptr);
490
491 // Create an iterator over the result of the above dataset
492 // This will trigger the creation of the Execution Tree and launch it.
493 std::shared_ptr<Iterator> iter = ds->CreateIterator();
494 EXPECT_NE(iter, nullptr);
495
496 // Iterate the dataset and get each row
497 std::unordered_map<std::string, mindspore::MSTensor> row;
498 ASSERT_OK(iter->GetNextRow(&row));
499
500 uint64_t i = 0;
501 while (row.size() != 0) {
502 auto ind = row["col1"];
503 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
504
505 ASSERT_OK(iter->GetNextRow(&row));
506 i++;
507 }
508
509 EXPECT_EQ(i, 3);
510
511 // Manually terminate the pipeline
512 iter->Stop();
513 GlobalContext::config_manager()->set_seed(curr_seed);
514 }
515
TEST_F(MindDataTestPipeline,TestRandomDatasetFloat)516 TEST_F(MindDataTestPipeline, TestRandomDatasetFloat) {
517 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetFloat.";
518
519 // Create a RandomDataset with Float numbers for given shape
520 u_int32_t curr_seed = GlobalContext::config_manager()->seed();
521 GlobalContext::config_manager()->set_seed(369);
522 std::shared_ptr<SchemaObj> schema = Schema();
523 ASSERT_OK(schema->add_column("col1", mindspore::DataType::kNumberTypeFloat16, {2, 3}));
524 std::shared_ptr<Dataset> ds = RandomData(4, schema);
525 EXPECT_NE(ds, nullptr);
526 ds = ds->SetNumWorkers(2);
527 EXPECT_NE(ds, nullptr);
528
529 // Create an iterator over the result of the above dataset
530 // This will trigger the creation of the Execution Tree and launch it.
531 std::shared_ptr<Iterator> iter = ds->CreateIterator();
532 EXPECT_NE(iter, nullptr);
533
534 // Iterate the dataset and get each row
535 std::unordered_map<std::string, mindspore::MSTensor> row;
536 ASSERT_OK(iter->GetNextRow(&row));
537
538 uint64_t i = 0;
539 while (row.size() != 0) {
540 auto ind = row["col1"];
541 TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
542
543 ASSERT_OK(iter->GetNextRow(&row));
544 i++;
545 }
546
547 EXPECT_EQ(i, 4);
548
549 // Manually terminate the pipeline
550 iter->Stop();
551 GlobalContext::config_manager()->set_seed(curr_seed);
552 }
553
TEST_F(MindDataTestPipeline,TestRandomDatasetDuplicateColumnName)554 TEST_F(MindDataTestPipeline, TestRandomDatasetDuplicateColumnName) {
555 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetDuplicateColumnName.";
556
557 // Create a RandomDataset
558 std::shared_ptr<SchemaObj> schema = Schema();
559 ASSERT_OK(schema->add_column("image", mindspore::DataType::kNumberTypeUInt8, {2}));
560 ASSERT_OK(schema->add_column("label", mindspore::DataType::kNumberTypeUInt8, {1}));
561 std::shared_ptr<Dataset> ds = RandomData(50, schema, {"image", "image"});
562 // Expect failure: duplicate column names
563 EXPECT_EQ(ds->CreateIterator(), nullptr);
564 }
565
TEST_F(MindDataTestPipeline,TestRandomDatasetFail)566 TEST_F(MindDataTestPipeline, TestRandomDatasetFail) {
567 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomDatasetFail.";
568 // this will fail because num_workers is greater than num_rows
569 std::shared_ptr<Dataset> ds = RandomData(3)->SetNumWorkers(5);
570 EXPECT_EQ(ds->CreateIterator(), nullptr);
571 }
572