1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18
19 using namespace mindspore::dataset;
20 using mindspore::dataset::DataType;
21 using mindspore::dataset::Tensor;
22 using mindspore::dataset::TensorShape;
23
24 class MindDataTestPipeline : public UT::DatasetOpTesting {
25 protected:
26 };
27
TEST_F(MindDataTestPipeline,TestUSPSTrainDataset)28 TEST_F(MindDataTestPipeline, TestUSPSTrainDataset) {
29 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDataset.";
30
31 // Create a USPS Train Dataset
32 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
33 std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
34 EXPECT_NE(ds, nullptr);
35
36 // Create an iterator over the result of the above dataset
37 // This will trigger the creation of the Execution Tree and launch it.
38 std::shared_ptr<Iterator> iter = ds->CreateIterator();
39 EXPECT_NE(iter, nullptr);
40
41 // Iterate the dataset and get each row
42 std::unordered_map<std::string, mindspore::MSTensor> row;
43 ASSERT_OK(iter->GetNextRow(&row));
44
45 EXPECT_NE(row.find("image"), row.end());
46 EXPECT_NE(row.find("label"), row.end());
47
48 uint64_t i = 0;
49 while (row.size() != 0) {
50 i++;
51 auto image = row["image"];
52 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
53 ASSERT_OK(iter->GetNextRow(&row));
54 }
55
56 EXPECT_EQ(i, 3);
57
58 // Manually terminate the pipeline
59 iter->Stop();
60 }
61
TEST_F(MindDataTestPipeline,TestUSPSTestDataset)62 TEST_F(MindDataTestPipeline, TestUSPSTestDataset) {
63 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTestDataset.";
64
65 // Create a USPS Test Dataset
66 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
67 std::shared_ptr<Dataset> ds = USPS(folder_path, "test");
68 EXPECT_NE(ds, nullptr);
69
70 // Create an iterator over the result of the above dataset
71 // This will trigger the creation of the Execution Tree and launch it.
72 std::shared_ptr<Iterator> iter = ds->CreateIterator();
73 EXPECT_NE(iter, nullptr);
74
75 // Iterate the dataset and get each row
76 std::unordered_map<std::string, mindspore::MSTensor> row;
77 ASSERT_OK(iter->GetNextRow(&row));
78
79 EXPECT_NE(row.find("image"), row.end());
80 EXPECT_NE(row.find("label"), row.end());
81
82 uint64_t i = 0;
83 while (row.size() != 0) {
84 i++;
85 auto image = row["image"];
86 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
87 ASSERT_OK(iter->GetNextRow(&row));
88 }
89
90 EXPECT_EQ(i, 3);
91
92 // Manually terminate the pipeline
93 iter->Stop();
94 }
95
TEST_F(MindDataTestPipeline,TestUSPSAllDataset)96 TEST_F(MindDataTestPipeline, TestUSPSAllDataset) {
97 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSAllDataset.";
98
99 // Create a USPS Test Dataset
100 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
101 std::shared_ptr<Dataset> ds = USPS(folder_path, "all");
102 EXPECT_NE(ds, nullptr);
103
104 // Create an iterator over the result of the above dataset
105 // This will trigger the creation of the Execution Tree and launch it.
106 std::shared_ptr<Iterator> iter = ds->CreateIterator();
107 EXPECT_NE(iter, nullptr);
108
109 // Iterate the dataset and get each row
110 std::unordered_map<std::string, mindspore::MSTensor> row;
111 ASSERT_OK(iter->GetNextRow(&row));
112
113 EXPECT_NE(row.find("image"), row.end());
114 EXPECT_NE(row.find("label"), row.end());
115
116 uint64_t i = 0;
117 while (row.size() != 0) {
118 i++;
119 auto image = row["image"];
120 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
121 ASSERT_OK(iter->GetNextRow(&row));
122 }
123
124 EXPECT_EQ(i, 6);
125
126 // Manually terminate the pipeline
127 iter->Stop();
128 }
129
TEST_F(MindDataTestPipeline,TestUSPSDatasetWithPipeline)130 TEST_F(MindDataTestPipeline, TestUSPSDatasetWithPipeline) {
131 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetWithPipeline.";
132
133 // Create two USPS Train Dataset
134 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
135 std::shared_ptr<Dataset> ds1 = USPS(folder_path, "train");
136 std::shared_ptr<Dataset> ds2 = USPS(folder_path, "train");
137 EXPECT_NE(ds1, nullptr);
138 EXPECT_NE(ds2, nullptr);
139
140 // Create two Repeat operation on ds
141 int32_t repeat_num = 1;
142 ds1 = ds1->Repeat(repeat_num);
143 EXPECT_NE(ds1, nullptr);
144 repeat_num = 1;
145 ds2 = ds2->Repeat(repeat_num);
146 EXPECT_NE(ds2, nullptr);
147
148 // Create two Project operation on ds
149 std::vector<std::string> column_project = {"image", "label"};
150 ds1 = ds1->Project(column_project);
151 EXPECT_NE(ds1, nullptr);
152 ds2 = ds2->Project(column_project);
153 EXPECT_NE(ds2, nullptr);
154
155 // Create a Concat operation on the ds
156 ds1 = ds1->Concat({ds2});
157 EXPECT_NE(ds1, nullptr);
158
159 // Create an iterator over the result of the above dataset
160 // This will trigger the creation of the Execution Tree and launch it.
161 std::shared_ptr<Iterator> iter = ds1->CreateIterator();
162 EXPECT_NE(iter, nullptr);
163
164 // Iterate the dataset and get each row
165 std::unordered_map<std::string, mindspore::MSTensor> row;
166 ASSERT_OK(iter->GetNextRow(&row));
167
168 EXPECT_NE(row.find("image"), row.end());
169 EXPECT_NE(row.find("label"), row.end());
170
171 uint64_t i = 0;
172 while (row.size() != 0) {
173 i++;
174 auto image = row["image"];
175 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
176 ASSERT_OK(iter->GetNextRow(&row));
177 }
178
179 EXPECT_EQ(i, 6);
180
181 // Manually terminate the pipeline
182 iter->Stop();
183 }
184
TEST_F(MindDataTestPipeline,TestGetUSPSDatasetSize)185 TEST_F(MindDataTestPipeline, TestGetUSPSDatasetSize) {
186 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestGetUSPSTrainDatasetSize.";
187
188 // Create a USPS Train Dataset
189 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
190 std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
191 EXPECT_NE(ds, nullptr);
192
193 EXPECT_EQ(ds->GetDatasetSize(), 3);
194 }
195
TEST_F(MindDataTestPipeline,TestUSPSDatasetGetters)196 TEST_F(MindDataTestPipeline, TestUSPSDatasetGetters) {
197 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSTrainDatasetGetters.";
198
199 // Create a USPS Train Dataset
200 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
201 std::shared_ptr<Dataset> ds = USPS(folder_path, "train");
202 EXPECT_NE(ds, nullptr);
203
204 EXPECT_EQ(ds->GetDatasetSize(), 3);
205 std::vector<DataType> types = ToDETypes(ds->GetOutputTypes());
206 std::vector<TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
207 std::vector<std::string> column_names = {"image", "label"};
208 EXPECT_EQ(types.size(), 2);
209 EXPECT_EQ(types[0].ToString(), "uint8");
210 EXPECT_EQ(types[1].ToString(), "uint32");
211 EXPECT_EQ(shapes.size(), 2);
212 EXPECT_EQ(shapes[0].ToString(), "<16,16,1>");
213 EXPECT_EQ(shapes[1].ToString(), "<>");
214 EXPECT_EQ(ds->GetBatchSize(), 1);
215 EXPECT_EQ(ds->GetRepeatCount(), 1);
216
217 EXPECT_EQ(ds->GetDatasetSize(), 3);
218 EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
219 EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
220
221 EXPECT_EQ(ds->GetColumnNames(), column_names);
222 EXPECT_EQ(ds->GetDatasetSize(), 3);
223 EXPECT_EQ(ToDETypes(ds->GetOutputTypes()), types);
224 EXPECT_EQ(ToTensorShapeVec(ds->GetOutputShapes()), shapes);
225 EXPECT_EQ(ds->GetBatchSize(), 1);
226 EXPECT_EQ(ds->GetRepeatCount(), 1);
227 EXPECT_EQ(ds->GetDatasetSize(), 3);
228 }
229
TEST_F(MindDataTestPipeline,TestUSPSDatasetFail)230 TEST_F(MindDataTestPipeline, TestUSPSDatasetFail) {
231 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetFail.";
232
233 // Create a USPS Dataset
234 std::shared_ptr<Dataset> ds = USPS("", "train");
235 EXPECT_NE(ds, nullptr);
236
237 // Create an iterator over the result of the above dataset
238 std::shared_ptr<Iterator> iter = ds->CreateIterator();
239 // Expect failure: invalid USPS input
240 EXPECT_EQ(iter, nullptr);
241 }
242
TEST_F(MindDataTestPipeline,TestUSPSDatasetWithInvalidUsageFail)243 TEST_F(MindDataTestPipeline, TestUSPSDatasetWithInvalidUsageFail) {
244 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUSPSDatasetWithInvalidUsageFail.";
245
246 // Create a USPS Dataset
247 std::string folder_path = datasets_root_path_ + "/testUSPSDataset/";
248 std::shared_ptr<Dataset> ds = USPS(folder_path, "validation");
249 EXPECT_NE(ds, nullptr);
250
251 // Create an iterator over the result of the above dataset
252 std::shared_ptr<Iterator> iter = ds->CreateIterator();
253 // Expect failure: invalid USPS input, validation is not a valid usage
254 EXPECT_EQ(iter, nullptr);
255 }
256