1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17
18 #include "minddata/dataset/include/dataset/datasets.h"
19
20 using namespace mindspore::dataset;
21 using mindspore::dataset::DataType;
22 using mindspore::dataset::Tensor;
23 using mindspore::dataset::TensorShape;
24
25 class MindDataTestPipeline : public UT::DatasetOpTesting {
26 protected:
27 };
28
TEST_F(MindDataTestPipeline,TestSBUDataset)29 TEST_F(MindDataTestPipeline, TestSBUDataset) {
30 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDataset.";
31
32 // Create a SBU Dataset
33 std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
34 std::shared_ptr<Dataset> ds = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
35 EXPECT_NE(ds, nullptr);
36
37 // Create an iterator over the result of the above dataset
38 // This will trigger the creation of the Execution Tree and launch it.
39 std::shared_ptr<Iterator> iter = ds->CreateIterator();
40 EXPECT_NE(iter, nullptr);
41
42 // Iterate the dataset and get each row
43 std::unordered_map<std::string, mindspore::MSTensor> row;
44 ASSERT_OK(iter->GetNextRow(&row));
45
46 EXPECT_NE(row.find("image"), row.end());
47 EXPECT_NE(row.find("caption"), row.end());
48
49 uint64_t i = 0;
50 while (row.size() != 0) {
51 i++;
52 auto image = row["image"];
53 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
54 ASSERT_OK(iter->GetNextRow(&row));
55 }
56
57 EXPECT_EQ(i, 5);
58
59 // Manually terminate the pipeline
60 iter->Stop();
61 }
62
TEST_F(MindDataTestPipeline,TestSBUDatasetWithPipeline)63 TEST_F(MindDataTestPipeline, TestSBUDatasetWithPipeline) {
64 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithPipeline.";
65
66 // Create two SBU Dataset
67 std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
68 std::shared_ptr<Dataset> ds1 = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
69 std::shared_ptr<Dataset> ds2 = SBU(folder_path, true, std::make_shared<RandomSampler>(false, 5));
70 EXPECT_NE(ds1, nullptr);
71 EXPECT_NE(ds2, nullptr);
72
73 // Create two Repeat operation on ds
74 int32_t repeat_num = 1;
75 ds1 = ds1->Repeat(repeat_num);
76 EXPECT_NE(ds1, nullptr);
77 repeat_num = 1;
78 ds2 = ds2->Repeat(repeat_num);
79 EXPECT_NE(ds2, nullptr);
80
81 // Create two Project operation on ds
82 std::vector<std::string> column_project = {"image", "caption"};
83 ds1 = ds1->Project(column_project);
84 EXPECT_NE(ds1, nullptr);
85 ds2 = ds2->Project(column_project);
86 EXPECT_NE(ds2, nullptr);
87
88 // Create a Concat operation on the ds
89 ds1 = ds1->Concat({ds2});
90 EXPECT_NE(ds1, nullptr);
91
92 // Create an iterator over the result of the above dataset
93 // This will trigger the creation of the Execution Tree and launch it.
94 std::shared_ptr<Iterator> iter = ds1->CreateIterator();
95 EXPECT_NE(iter, nullptr);
96
97 // Iterate the dataset and get each row
98 std::unordered_map<std::string, mindspore::MSTensor> row;
99 ASSERT_OK(iter->GetNextRow(&row));
100
101 EXPECT_NE(row.find("image"), row.end());
102 EXPECT_NE(row.find("caption"), row.end());
103
104 uint64_t i = 0;
105 while (row.size() != 0) {
106 i++;
107 auto image = row["image"];
108 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
109 ASSERT_OK(iter->GetNextRow(&row));
110 }
111
112 EXPECT_EQ(i, 10);
113
114 // Manually terminate the pipeline
115 iter->Stop();
116 }
117
TEST_F(MindDataTestPipeline,TestGetSBUDatasetSize)118 TEST_F(MindDataTestPipeline, TestGetSBUDatasetSize) {
119 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetSBUDatasetSize.";
120
121 // Create a SBU Dataset
122 std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
123 std::shared_ptr<Dataset> ds = SBU(folder_path, true);
124 EXPECT_NE(ds, nullptr);
125
126 EXPECT_EQ(ds->GetDatasetSize(), 5);
127 }
128
TEST_F(MindDataTestPipeline,TestSBUDatasetGetters)129 TEST_F(MindDataTestPipeline, TestSBUDatasetGetters) {
130 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetGetters.";
131
132 // Create a SBU Dataset
133 std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
134 std::shared_ptr<Dataset> ds = SBU(folder_path, true);
135 EXPECT_NE(ds, nullptr);
136
137 EXPECT_EQ(ds->GetDatasetSize(), 5);
138 std::vector<DataType> types = ToDETypes(ds->GetOutputTypes());
139 std::vector<TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
140 std::vector<std::string> column_names = {"image", "caption"};
141 EXPECT_EQ(types.size(), 2);
142 EXPECT_EQ(types[0].ToString(), "uint8");
143 EXPECT_EQ(types[1].ToString(), "string");
144
145 EXPECT_EQ(ds->GetBatchSize(), 1);
146 EXPECT_EQ(ds->GetRepeatCount(), 1);
147
148 EXPECT_EQ(ds->GetDatasetSize(), 5);
149 EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
150 EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
151 EXPECT_EQ(ds->GetNumClasses(), -1);
152
153 EXPECT_EQ(ds->GetColumnNames(), column_names);
154 EXPECT_EQ(ds->GetDatasetSize(), 5);
155 EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
156 EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
157 EXPECT_EQ(ds->GetBatchSize(), 1);
158 EXPECT_EQ(ds->GetRepeatCount(), 1);
159 EXPECT_EQ(ds->GetNumClasses(), -1);
160 EXPECT_EQ(ds->GetDatasetSize(), 5);
161 }
162
TEST_F(MindDataTestPipeline,TestSBUDatasetFail)163 TEST_F(MindDataTestPipeline, TestSBUDatasetFail) {
164 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetFail.";
165
166 // Create a SBU Dataset
167 std::shared_ptr<Dataset> ds = SBU("", true, std::make_shared<RandomSampler>(false, 10));
168 EXPECT_NE(ds, nullptr);
169
170 // Create an iterator over the result of the above dataset
171 std::shared_ptr<Iterator> iter = ds->CreateIterator();
172 // Expect failure: invalid SBU input
173 EXPECT_EQ(iter, nullptr);
174 }
175
TEST_F(MindDataTestPipeline,TestSBUDatasetWithNullSamplerFail)176 TEST_F(MindDataTestPipeline, TestSBUDatasetWithNullSamplerFail) {
177 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSBUDatasetWithNullSamplerFail.";
178
179 // Create a SBU Dataset
180 std::string folder_path = datasets_root_path_ + "/testSBUDataset/";
181 std::shared_ptr<Dataset> ds = SBU(folder_path, true, nullptr);
182 EXPECT_NE(ds, nullptr);
183
184 // Create an iterator over the result of the above dataset
185 std::shared_ptr<Iterator> iter = ds->CreateIterator();
186 // Expect failure: invalid SBU input, sampler cannot be nullptr
187 EXPECT_EQ(iter, nullptr);
188 }
189