1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18 #include "minddata/dataset/include/dataset/vision.h"
19 #include "minddata/dataset/core/global_context.h"
20
21 using namespace mindspore::dataset;
22
23 using mindspore::dataset::DataType;
24 using mindspore::dataset::ShuffleMode;
25 using mindspore::dataset::TensorShape;
26
27 class MindDataTestPipeline : public UT::DatasetOpTesting {
28 protected:
29 };
30
TEST_F(MindDataTestPipeline,TestTFRecordDatasetBasic)31 TEST_F(MindDataTestPipeline, TestTFRecordDatasetBasic) {
32 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetBasic.";
33
34 // Create a TFRecord Dataset
35 std::string file_path = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
36 std::string schema_path = datasets_root_path_ + "/test_tf_file_3_images2/datasetSchema.json";
37 std::shared_ptr<Dataset> ds = TFRecord({file_path}, schema_path, {"image"}, 0);
38 EXPECT_NE(ds, nullptr);
39
40 // Create a Repeat operation on ds
41 int32_t repeat_num = 2;
42 ds = ds->Repeat(repeat_num);
43 EXPECT_NE(ds, nullptr);
44
45 // Create objects for the tensor ops
46 std::shared_ptr<TensorTransform> decode_op = std::make_shared<vision::Decode>();
47 std::shared_ptr<TensorTransform> random_horizontal_flip_op = std::make_shared<vision::RandomHorizontalFlip>(0.5);
48 EXPECT_NE(random_horizontal_flip_op, nullptr);
49
50 // Create a Map operation on ds
51 ds = ds->Map({decode_op, random_horizontal_flip_op}, {}, {}, {"image"});
52 EXPECT_NE(ds, nullptr);
53
54 // Create a Batch operation on ds
55 int32_t batch_size = 1;
56 ds = ds->Batch(batch_size);
57 EXPECT_NE(ds, nullptr);
58
59 // Create an iterator over the result of the above dataset
60 // This will trigger the creation of the Execution Tree and launch it.
61 std::shared_ptr<Iterator> iter = ds->CreateIterator();
62 EXPECT_NE(iter, nullptr);
63
64 // Iterate the dataset and get each row
65 std::unordered_map<std::string, mindspore::MSTensor> row;
66 ASSERT_OK(iter->GetNextRow(&row));
67
68 // Check column
69 EXPECT_EQ(row.size(), 1);
70 EXPECT_NE(row.find("image"), row.end());
71
72 uint64_t i = 0;
73 while (row.size() != 0) {
74 auto image = row["image"];
75
76 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
77 ASSERT_OK(iter->GetNextRow(&row));
78 i++;
79 }
80
81 EXPECT_EQ(i, 6);
82
83 // Manually terminate the pipeline
84 iter->Stop();
85 }
86
TEST_F(MindDataTestPipeline,TestTFRecordDatasetBasicGetters)87 TEST_F(MindDataTestPipeline, TestTFRecordDatasetBasicGetters) {
88 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetBasicGetters.";
89
90 // Create a TFRecord Dataset
91 std::string file_path = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
92 std::string schema_path = datasets_root_path_ + "/test_tf_file_3_images2/datasetSchema.json";
93 std::shared_ptr<Dataset> ds = TFRecord({file_path}, schema_path, {"image"}, 0);
94 EXPECT_NE(ds, nullptr);
95
96 // Create a Repeat operation on ds
97 int32_t repeat_num = 2;
98 ds = ds->Repeat(repeat_num);
99 EXPECT_NE(ds, nullptr);
100
101 // Create objects for the tensor ops
102 std::shared_ptr<TensorTransform> random_horizontal_flip_op = std::make_shared<vision::RandomHorizontalFlip>(0.5);
103 EXPECT_NE(random_horizontal_flip_op, nullptr);
104
105 // Create a Map operation on ds
106 ds = ds->Map({random_horizontal_flip_op}, {}, {}, {"image"});
107 EXPECT_NE(ds, nullptr);
108
109 // Create a Batch operation on ds
110 int32_t batch_size = 1;
111 ds = ds->Batch(batch_size);
112 EXPECT_NE(ds, nullptr);
113
114 EXPECT_EQ(ds->GetDatasetSize(), 6);
115 std::vector<std::string> column_names = {"image"};
116 EXPECT_EQ(ds->GetColumnNames(), column_names);
117 }
118
TEST_F(MindDataTestPipeline,TestTFRecordDatasetShuffle)119 TEST_F(MindDataTestPipeline, TestTFRecordDatasetShuffle) {
120 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetShuffle.";
121 // This case is to verify if the list of datafiles are sorted in lexicographical order.
122 // Set configuration
123 uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
124 MS_LOG(DEBUG) << "ORIGINAL num_parallel_workers: " << original_num_parallel_workers;
125 GlobalContext::config_manager()->set_num_parallel_workers(1);
126
127 // Create a TFRecord Dataset
128 std::string file1 = datasets_root_path_ + "/tf_file_dataset/test1.data";
129 std::string file2 = datasets_root_path_ + "/tf_file_dataset/test2.data";
130 std::string file3 = datasets_root_path_ + "/tf_file_dataset/test3.data";
131 std::string file4 = datasets_root_path_ + "/tf_file_dataset/test4.data";
132 std::shared_ptr<Dataset> ds1 = TFRecord({file4, file3, file2, file1}, "", {"scalars"}, 0, ShuffleMode::kFalse);
133 EXPECT_NE(ds1, nullptr);
134 std::shared_ptr<Dataset> ds2 = TFRecord({file1}, "", {"scalars"}, 0, ShuffleMode::kFalse);
135 EXPECT_NE(ds2, nullptr);
136
137 // Create an iterator over the result of the above dataset
138 // This will trigger the creation of the Execution Tree and launch it.
139 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
140 EXPECT_NE(iter1, nullptr);
141 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
142 EXPECT_NE(iter2, nullptr);
143
144 // Iterate the dataset and get each row
145 std::unordered_map<std::string, mindspore::MSTensor> row1;
146 ASSERT_OK(iter1->GetNextRow(&row1));
147 std::unordered_map<std::string, mindspore::MSTensor> row2;
148 ASSERT_OK(iter2->GetNextRow(&row2));
149
150 uint64_t i = 0;
151 int64_t value1 = 0;
152 int64_t value2 = 0;
153 while (row1.size() != 0 && row2.size() != 0) {
154 auto scalars1 = row1["scalars"];
155 std::shared_ptr<Tensor> de_scalars1;
156 ASSERT_OK(Tensor::CreateFromMSTensor(scalars1, &de_scalars1));
157 ASSERT_OK(de_scalars1->GetItemAt(&value1, {0}));
158
159 auto scalars2 = row2["scalars"];
160 std::shared_ptr<Tensor> de_scalars2;
161 ASSERT_OK(Tensor::CreateFromMSTensor(scalars2, &de_scalars2));
162 ASSERT_OK(de_scalars2->GetItemAt(&value2, {0}));
163
164 EXPECT_EQ(value1, value2);
165
166 ASSERT_OK(iter1->GetNextRow(&row1));
167 ASSERT_OK(iter2->GetNextRow(&row2));
168 i++;
169 }
170 EXPECT_EQ(i, 10);
171 // Manually terminate the pipeline
172 iter1->Stop();
173 iter2->Stop();
174
175 // Restore configuration
176 GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
177 }
178
TEST_F(MindDataTestPipeline,TestTFRecordDatasetShuffle2)179 TEST_F(MindDataTestPipeline, TestTFRecordDatasetShuffle2) {
180 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetShuffle2.";
181 // This case is to verify the content of the data is indeed shuffled.
182 // Set configuration
183 uint32_t original_seed = GlobalContext::config_manager()->seed();
184 uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers();
185 MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers;
186 GlobalContext::config_manager()->set_seed(155);
187 GlobalContext::config_manager()->set_num_parallel_workers(1);
188
189 // Create a TFRecord Dataset
190 std::string file = datasets_root_path_ + "/tf_file_dataset/test1.data";
191 std::shared_ptr<Dataset> ds = TFRecord({file}, nullptr, {"scalars"}, 0, ShuffleMode::kGlobal);
192 EXPECT_NE(ds, nullptr);
193
194 // Create an iterator over the result of the above dataset
195 // This will trigger the creation of the Execution Tree and launch it.
196 std::shared_ptr<Iterator> iter = ds->CreateIterator();
197 EXPECT_NE(iter, nullptr);
198
199 // Iterate the dataset and get each row
200 std::unordered_map<std::string, mindspore::MSTensor> row;
201 ASSERT_OK(iter->GetNextRow(&row));
202
203 std::vector<int> expect = {9, 3, 4, 7, 2, 1, 6, 8, 10, 5};
204 std::vector<int> actual = {};
205 int64_t value = 0;
206 uint64_t i = 0;
207 while (row.size() != 0) {
208 auto scalars = row["scalars"];
209 std::shared_ptr<Tensor> de_scalars;
210 ASSERT_OK(Tensor::CreateFromMSTensor(scalars, &de_scalars));
211 ASSERT_OK(de_scalars->GetItemAt(&value, {0}));
212 actual.push_back(value);
213
214 ASSERT_OK(iter->GetNextRow(&row));
215 i++;
216 }
217 ASSERT_EQ(actual, expect);
218 EXPECT_EQ(i, 10);
219 // Manually terminate the pipeline
220 iter->Stop();
221
222 // Restore configuration
223 GlobalContext::config_manager()->set_seed(original_seed);
224 GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers);
225 }
226
TEST_F(MindDataTestPipeline,TestTFRecordDatasetSchemaPath)227 TEST_F(MindDataTestPipeline, TestTFRecordDatasetSchemaPath) {
228 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetSchemaPath.";
229
230 // Create a TFRecord Dataset
231 std::string file_path1 = datasets_root_path_ + "/testTFTestAllTypes/test.data";
232 std::string file_path2 = datasets_root_path_ + "/testTFTestAllTypes/test2.data";
233 std::string schema_path = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
234 std::shared_ptr<Dataset> ds = TFRecord({file_path2, file_path1}, schema_path, {}, 9, ShuffleMode::kFalse);
235 EXPECT_NE(ds, nullptr);
236
237 // Create an iterator over the result of the above dataset
238 // This will trigger the creation of the Execution Tree and launch it.
239 std::shared_ptr<Iterator> iter = ds->CreateIterator();
240 EXPECT_NE(iter, nullptr);
241
242 // Iterate the dataset and get each row
243 std::unordered_map<std::string, mindspore::MSTensor> row;
244 ASSERT_OK(iter->GetNextRow(&row));
245
246 // Check column
247 EXPECT_EQ(row.size(), 8);
248 EXPECT_NE(row.find("col_sint16"), row.end());
249 EXPECT_NE(row.find("col_sint32"), row.end());
250 EXPECT_NE(row.find("col_sint64"), row.end());
251 EXPECT_NE(row.find("col_float"), row.end());
252 EXPECT_NE(row.find("col_1d"), row.end());
253 EXPECT_NE(row.find("col_2d"), row.end());
254 EXPECT_NE(row.find("col_3d"), row.end());
255 EXPECT_NE(row.find("col_binary"), row.end());
256
257 uint64_t i = 0;
258 while (row.size() != 0) {
259 i++;
260 ASSERT_OK(iter->GetNextRow(&row));
261 }
262
263 EXPECT_EQ(i, 9);
264
265 // Manually terminate the pipeline
266 iter->Stop();
267 }
268
TEST_F(MindDataTestPipeline,TestTFRecordDatasetSchemaObj)269 TEST_F(MindDataTestPipeline, TestTFRecordDatasetSchemaObj) {
270 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetSchemaObj.";
271
272 // Create a TFRecord Dataset
273 std::string file_path = datasets_root_path_ + "/testTFTestAllTypes/test.data";
274 std::shared_ptr<SchemaObj> schema = Schema();
275 ASSERT_OK(schema->add_column("col_sint16", "int16", {1}));
276 ASSERT_OK(schema->add_column("col_float", "float32", {1}));
277 ASSERT_OK(schema->add_column("col_2d", "int64", {2, 2}));
278 std::shared_ptr<Dataset> ds = TFRecord({file_path}, schema);
279 EXPECT_NE(ds, nullptr);
280
281 // Create an iterator over the result of the above dataset
282 // This will trigger the creation of the Execution Tree and launch it.
283 std::shared_ptr<Iterator> iter = ds->CreateIterator();
284 EXPECT_NE(iter, nullptr);
285
286 // Iterate the dataset and get each row
287 std::unordered_map<std::string, mindspore::MSTensor> row;
288 ASSERT_OK(iter->GetNextRow(&row));
289
290 // Check column
291 EXPECT_EQ(row.size(), 3);
292 EXPECT_NE(row.find("col_sint16"), row.end());
293 EXPECT_NE(row.find("col_float"), row.end());
294 EXPECT_NE(row.find("col_2d"), row.end());
295
296 std::vector<int64_t> expect_num = {1};
297 std::vector<int64_t> expect_2d = {2, 2};
298
299 uint64_t i = 0;
300 while (row.size() != 0) {
301 auto col_sint16 = row["col_sint16"];
302 auto col_float = row["col_float"];
303 auto col_2d = row["col_2d"];
304
305 // Validate shape
306 ASSERT_EQ(col_sint16.Shape(), expect_num);
307 ASSERT_EQ(col_float.Shape(), expect_num);
308 ASSERT_EQ(col_2d.Shape(), expect_2d);
309
310 // Validate Rank
311 ASSERT_EQ(col_sint16.Shape().size(), 1);
312 ASSERT_EQ(col_float.Shape().size(), 1);
313 ASSERT_EQ(col_2d.Shape().size(), 2);
314
315 // Validate type
316 ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16);
317 ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32);
318 ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64);
319
320 ASSERT_OK(iter->GetNextRow(&row));
321 i++;
322 }
323
324 EXPECT_EQ(i, 12);
325
326 // Manually terminate the pipeline
327 iter->Stop();
328 }
329
TEST_F(MindDataTestPipeline,TestTFRecordDatasetNoSchema)330 TEST_F(MindDataTestPipeline, TestTFRecordDatasetNoSchema) {
331 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetNoSchema.";
332
333 // Create a TFRecord Dataset
334 std::string file_path = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
335 std::shared_ptr<SchemaObj> schema = nullptr;
336 std::shared_ptr<Dataset> ds = TFRecord({file_path}, nullptr, {});
337 EXPECT_NE(ds, nullptr);
338
339 // Create an iterator over the result of the above dataset
340 // This will trigger the creation of the Execution Tree and launch it.
341 std::shared_ptr<Iterator> iter = ds->CreateIterator();
342 EXPECT_NE(iter, nullptr);
343
344 // Iterate the dataset and get each row
345 std::unordered_map<std::string, mindspore::MSTensor> row;
346 ASSERT_OK(iter->GetNextRow(&row));
347
348 // Check column
349 EXPECT_EQ(row.size(), 2);
350 EXPECT_NE(row.find("image"), row.end());
351 EXPECT_NE(row.find("label"), row.end());
352
353 uint64_t i = 0;
354 while (row.size() != 0) {
355 auto image = row["image"];
356 auto label = row["label"];
357
358 MS_LOG(INFO) << "Shape of column [image]:" << image.Shape();
359 MS_LOG(INFO) << "Shape of column [label]:" << label.Shape();
360 ASSERT_OK(iter->GetNextRow(&row));
361 i++;
362 }
363
364 EXPECT_EQ(i, 3);
365
366 // Manually terminate the pipeline
367 iter->Stop();
368 }
369
TEST_F(MindDataTestPipeline,TestTFRecordDatasetColName)370 TEST_F(MindDataTestPipeline, TestTFRecordDatasetColName) {
371 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetColName.";
372
373 // Create a TFRecord Dataset
374 // The dataset has two columns("image", "label") and 3 rows
375 std::string file_path = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
376 std::shared_ptr<Dataset> ds = TFRecord({file_path}, "", {"image"}, 0);
377 EXPECT_NE(ds, nullptr);
378
379 // Create an iterator over the result of the above dataset
380 // This will trigger the creation of the Execution Tree and launch it.
381 std::shared_ptr<Iterator> iter = ds->CreateIterator();
382 EXPECT_NE(iter, nullptr);
383
384 // Iterate the dataset and get each row
385 std::unordered_map<std::string, mindspore::MSTensor> row;
386 ASSERT_OK(iter->GetNextRow(&row));
387
388 // Check column
389 EXPECT_EQ(row.size(), 1);
390 EXPECT_NE(row.find("image"), row.end());
391
392 uint64_t i = 0;
393 while (row.size() != 0) {
394 i++;
395 ASSERT_OK(iter->GetNextRow(&row));
396 }
397
398 EXPECT_EQ(i, 3);
399
400 // Manually terminate the pipeline
401 iter->Stop();
402 }
403
TEST_F(MindDataTestPipeline,TestTFRecordDatasetShard)404 TEST_F(MindDataTestPipeline, TestTFRecordDatasetShard) {
405 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetShard.";
406
407 // Create a TFRecord Dataset
408 // Each file has two columns("image", "label") and 3 rows
409 std::vector<std::string> files = {datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data",
410 datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0002.data",
411 datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0003.data"};
412 std::shared_ptr<Dataset> ds1 = TFRecord(files, "", {}, 0, ShuffleMode::kFalse, 2, 1, true);
413 EXPECT_NE(ds1, nullptr);
414 std::shared_ptr<Dataset> ds2 = TFRecord(files, "", {}, 0, ShuffleMode::kFalse, 2, 1, false);
415 EXPECT_NE(ds2, nullptr);
416
417 // Create an iterator over the result of the above dataset
418 // This will trigger the creation of the Execution Tree and launch it.
419 std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
420 EXPECT_NE(iter1, nullptr);
421 std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
422 EXPECT_NE(iter2, nullptr);
423
424 // Iterate the dataset and get each row
425 std::unordered_map<std::string, mindspore::MSTensor> row1;
426 ASSERT_OK(iter1->GetNextRow(&row1));
427 std::unordered_map<std::string, mindspore::MSTensor> row2;
428 ASSERT_OK(iter2->GetNextRow(&row2));
429
430 uint64_t i = 0;
431 uint64_t j = 0;
432 while (row1.size() != 0) {
433 i++;
434 ASSERT_OK(iter1->GetNextRow(&row1));
435 }
436
437 while (row2.size() != 0) {
438 j++;
439 ASSERT_OK(iter2->GetNextRow(&row2));
440 }
441
442 EXPECT_EQ(i, 5);
443 EXPECT_EQ(j, 3);
444 // Manually terminate the pipeline
445 iter1->Stop();
446 iter2->Stop();
447 }
448
TEST_F(MindDataTestPipeline,TestTFRecordDatasetExeception)449 TEST_F(MindDataTestPipeline, TestTFRecordDatasetExeception) {
450 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetExeception.";
451
452 // This case expected to fail because the list of dir_path cannot be empty.
453 std::shared_ptr<Dataset> ds1 = TFRecord({});
454 EXPECT_EQ(ds1->CreateIterator(), nullptr);
455
456 // This case expected to fail because the file in dir_path is not exist.
457 std::string file_path = datasets_root_path_ + "/testTFTestAllTypes/test.data";
458 std::shared_ptr<Dataset> ds2 = TFRecord({file_path, "noexist.data"});
459 EXPECT_EQ(ds2->CreateIterator(), nullptr);
460
461 // This case expected to fail because the file of schema is not exist.
462 std::shared_ptr<Dataset> ds4 = TFRecord({file_path, "notexist.json"});
463 EXPECT_EQ(ds4->CreateIterator(), nullptr);
464
465 // This case expected to fail because num_samples is negative.
466 std::shared_ptr<Dataset> ds5 = TFRecord({file_path}, "", {}, -1);
467 EXPECT_EQ(ds5->CreateIterator(), nullptr);
468
469 // This case expected to fail because num_shards is negative.
470 std::shared_ptr<Dataset> ds6 = TFRecord({file_path}, "", {}, 10, ShuffleMode::kFalse, 0);
471 EXPECT_EQ(ds6->CreateIterator(), nullptr);
472
473 // This case expected to fail because shard_id is out_of_bound.
474 std::shared_ptr<Dataset> ds7 = TFRecord({file_path}, "", {}, 10, ShuffleMode::kFalse, 3, 3);
475 EXPECT_EQ(ds7->CreateIterator(), nullptr);
476
477 // This case expected to fail because the provided number of files < num_shards in file-based sharding.
478 std::string file_path1 = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
479 std::string file_path2 = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0002.data";
480 std::shared_ptr<Dataset> ds8 = TFRecord({file_path1, file_path2}, "", {}, 0, ShuffleMode::kFalse, 3);
481 EXPECT_EQ(ds8->CreateIterator(), nullptr);
482 }
483
TEST_F(MindDataTestPipeline,TestTFRecordDatasetExeception2)484 TEST_F(MindDataTestPipeline, TestTFRecordDatasetExeception2) {
485 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTFRecordDatasetExeception2.";
486 // This case expected to fail because the input column name does not exist.
487
488 std::string file_path1 = datasets_root_path_ + "/testTFTestAllTypes/test.data";
489 std::string schema_path = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
490 // Create a TFRecord Dataset
491 // Column "image" does not exist in the dataset
492 std::shared_ptr<Dataset> ds = TFRecord({file_path1}, schema_path, {"image"}, 10);
493 EXPECT_NE(ds, nullptr);
494
495 // Create an iterator over the result of the above dataset
496 // This attempts to create Execution Tree and launch it.
497 std::shared_ptr<Iterator> iter = ds->CreateIterator();
498 EXPECT_EQ(iter, nullptr);
499 }
500
TEST_F(MindDataTestPipeline,TestIncorrectTFSchemaObject)501 TEST_F(MindDataTestPipeline, TestIncorrectTFSchemaObject) {
502 std::string path = datasets_root_path_ + "/test_tf_file_3_images2/train-0000-of-0001.data";
503 std::shared_ptr<SchemaObj> schema = Schema();
504 ASSERT_OK(schema->add_column("image", "uint8", {1}));
505 ASSERT_OK(schema->add_column("label", "int64", {1}));
506 std::shared_ptr<Dataset> ds = TFRecord({path}, schema);
507 EXPECT_NE(ds, nullptr);
508 auto itr = ds->CreateIterator();
509 EXPECT_NE(itr, nullptr);
510 MSTensorMap mp;
511 // This will fail due to the incorrect schema used
512 EXPECT_ERROR(itr->GetNextRow(&mp));
513 }
514
TEST_F(MindDataTestPipeline,TestIncorrectTFrecordFile)515 TEST_F(MindDataTestPipeline, TestIncorrectTFrecordFile) {
516 std::string path = datasets_root_path_ + "/test_tf_file_3_images2/datasetSchema.json";
517 std::shared_ptr<Dataset> ds = TFRecord({path});
518 EXPECT_NE(ds, nullptr);
519 // The tf record file is incorrect, hence validate param will fail
520 auto itr = ds->CreateIterator();
521 EXPECT_EQ(itr, nullptr);
522 }
523