1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18
19 using namespace mindspore::dataset;
20 using mindspore::dataset::Tensor;
21
22 class MindDataTestPipeline : public UT::DatasetOpTesting {
23 protected:
24 };
25
TEST_F(MindDataTestPipeline,TestAlbumBasic)26 TEST_F(MindDataTestPipeline, TestAlbumBasic) {
27 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasic.";
28
29 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
30 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
31 std::vector<std::string> column_names = {"image", "label", "id"};
32 // Create a Album Dataset
33 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
34 EXPECT_NE(ds, nullptr);
35
36 // Create an iterator over the result of the above dataset
37 // This will trigger the creation of the Execution Tree and launch it.
38 std::shared_ptr<Iterator> iter = ds->CreateIterator();
39 EXPECT_NE(iter, nullptr);
40
41 // Iterate the dataset and get each row
42 std::unordered_map<std::string, mindspore::MSTensor> row;
43 ASSERT_OK(iter->GetNextRow(&row));
44
45 uint64_t i = 0;
46 while (row.size() != 0) {
47 i++;
48 auto image = row["image"];
49 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
50 ASSERT_OK(iter->GetNextRow(&row));
51 }
52
53 EXPECT_EQ(i, 7);
54
55 // Manually terminate the pipeline
56 iter->Stop();
57 }
58
TEST_F(MindDataTestPipeline,TestAlbumBasicWithPipeline)59 TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
60 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumBasicWithPipeline.";
61
62 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
63 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
64 std::vector<std::string> column_names = {"image", "label", "id"};
65
66 // Create two Album Dataset
67 std::shared_ptr<Dataset> ds1 = Album(folder_path, schema_file, column_names);
68 std::shared_ptr<Dataset> ds2 = Album(folder_path, schema_file, column_names);
69 EXPECT_NE(ds1, nullptr);
70 EXPECT_NE(ds2, nullptr);
71
72 // Create two Repeat operation on ds
73 int32_t repeat_num = 2;
74 ds1 = ds1->Repeat(repeat_num);
75 EXPECT_NE(ds1, nullptr);
76 repeat_num = 3;
77 ds2 = ds2->Repeat(repeat_num);
78 EXPECT_NE(ds2, nullptr);
79
80 // Create two Project operation on ds
81 std::vector<std::string> column_project = {"image"};
82 ds1 = ds1->Project(column_project);
83 EXPECT_NE(ds1, nullptr);
84 ds2 = ds2->Project(column_project);
85 EXPECT_NE(ds2, nullptr);
86
87 // Create a Concat operation on the ds
88 ds1 = ds1->Concat({ds2});
89 EXPECT_NE(ds1, nullptr);
90
91 // Create an iterator over the result of the above dataset
92 // This will trigger the creation of the Execution Tree and launch it.
93 std::shared_ptr<Iterator> iter = ds1->CreateIterator();
94 EXPECT_NE(iter, nullptr);
95
96 // Iterate the dataset and get each row
97 std::unordered_map<std::string, mindspore::MSTensor> row;
98 ASSERT_OK(iter->GetNextRow(&row));
99
100 uint64_t i = 0;
101 while (row.size() != 0) {
102 i++;
103 auto image = row["image"];
104 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
105 ASSERT_OK(iter->GetNextRow(&row));
106 }
107
108 EXPECT_EQ(i, 35);
109
110 // Manually terminate the pipeline
111 iter->Stop();
112 }
113
TEST_F(MindDataTestPipeline,TestAlbumGetters)114 TEST_F(MindDataTestPipeline, TestAlbumGetters) {
115 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumGetters.";
116
117 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
118 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
119 std::vector<std::string> column_names = {"image", "label", "id"};
120 // Create a Album Dataset
121 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names);
122 EXPECT_NE(ds, nullptr);
123
124 int64_t num_classes = ds->GetNumClasses();
125 EXPECT_EQ(num_classes, -1);
126 int64_t num_samples = ds->GetDatasetSize();
127 EXPECT_EQ(num_samples, 7);
128
129 int64_t batch_size = ds->GetBatchSize();
130 EXPECT_EQ(batch_size, 1);
131 int64_t repeat_count = ds->GetRepeatCount();
132 EXPECT_EQ(repeat_count, 1);
133 EXPECT_EQ(ds->GetColumnNames(), column_names);
134
135 // Test get dataset size with num_samples > files in dataset
136 auto sampler = std::make_shared<SequentialSampler>(0, 12);
137 std::shared_ptr<Dataset> ds2 = Album(folder_path, schema_file, column_names, false, sampler);
138 num_samples = ds->GetDatasetSize();
139 EXPECT_EQ(num_samples, 7);
140 }
141
TEST_F(MindDataTestPipeline,TestAlbumDecode)142 TEST_F(MindDataTestPipeline, TestAlbumDecode) {
143 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDecode.";
144 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
145 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
146 std::vector<std::string> column_names = {"image", "label", "id"};
147 // Create a Album Dataset
148 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true);
149 EXPECT_NE(ds, nullptr);
150
151 // Create an iterator over the result of the above dataset
152 // This will trigger the creation of the Execution Tree and launch it.
153 std::shared_ptr<Iterator> iter = ds->CreateIterator();
154 EXPECT_NE(iter, nullptr);
155
156 // Iterate the dataset and get each row
157 std::unordered_map<std::string, mindspore::MSTensor> row;
158 ASSERT_OK(iter->GetNextRow(&row));
159
160 uint64_t i = 0;
161 while (row.size() != 0) {
162 i++;
163 auto image = row["image"];
164 auto shape = image.Shape();
165 MS_LOG(INFO) << "Tensor image shape size: " << shape.size();
166 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
167 EXPECT_GT(shape.size(), 1); // Verify decode=true took effect
168 ASSERT_OK(iter->GetNextRow(&row));
169 }
170
171 EXPECT_EQ(i, 7);
172
173 // Manually terminate the pipeline
174 iter->Stop();
175 }
176
TEST_F(MindDataTestPipeline,TestAlbumNumSamplers)177 TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
178 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumNumSamplers.";
179
180 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
181 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
182 std::vector<std::string> column_names = {"image", "label", "id"};
183 // Create a Album Dataset
184 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
185 EXPECT_NE(ds, nullptr);
186
187 // Create an iterator over the result of the above dataset
188 // This will trigger the creation of the Execution Tree and launch it.
189 std::shared_ptr<Iterator> iter = ds->CreateIterator();
190 EXPECT_NE(iter, nullptr);
191
192 // Iterate the dataset and get each row
193 std::unordered_map<std::string, mindspore::MSTensor> row;
194 ASSERT_OK(iter->GetNextRow(&row));
195
196 uint64_t i = 0;
197 while (row.size() != 0) {
198 i++;
199 auto image = row["image"];
200 MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
201 ASSERT_OK(iter->GetNextRow(&row));
202 }
203
204 EXPECT_EQ(i, 1);
205
206 // Manually terminate the pipeline
207 iter->Stop();
208 }
209
TEST_F(MindDataTestPipeline,TestAlbumError)210 TEST_F(MindDataTestPipeline, TestAlbumError) {
211 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumError.";
212 std::string folder_path = datasets_root_path_ + "/testAlbum/ima";
213 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
214 std::vector<std::string> column_names = {"image", "label", "id"};
215 // Create an Album Dataset
216 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, std::make_shared<SequentialSampler>(0, 1));
217 EXPECT_NE(ds, nullptr);
218
219 // Create an iterator over the result of the above dataset
220 std::shared_ptr<Iterator> iter = ds->CreateIterator();
221 // Expect failure: invalid Album input
222 EXPECT_EQ(iter, nullptr);
223 }
224
TEST_F(MindDataTestPipeline,TestAlbumWithNullSamplerError)225 TEST_F(MindDataTestPipeline, TestAlbumWithNullSamplerError) {
226 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumWithNullSamplerError.";
227 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
228 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
229 std::vector<std::string> column_names = {"image", "label", "id"};
230 // Create an Album Dataset
231 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true, nullptr);
232 EXPECT_NE(ds, nullptr);
233
234 // Create an iterator over the result of the above dataset
235 std::shared_ptr<Iterator> iter = ds->CreateIterator();
236 // Expect failure: invalid Album input, sampler cannot be nullptr
237 EXPECT_EQ(iter, nullptr);
238 }
239
TEST_F(MindDataTestPipeline,TestAlbumDuplicateColumnNameError)240 TEST_F(MindDataTestPipeline, TestAlbumDuplicateColumnNameError) {
241 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDuplicateColumnNameError.";
242 std::string folder_path = datasets_root_path_ + "/testAlbum/images";
243 std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json";
244 std::vector<std::string> column_names = {"image", "image", "id"};
245 // Create an Album Dataset
246 std::shared_ptr<Dataset> ds = Album(folder_path, schema_file, column_names, true);
247 EXPECT_NE(ds, nullptr);
248
249 // Create an iterator over the result of the above dataset
250 std::shared_ptr<Iterator> iter = ds->CreateIterator();
251 // Expect failure: invalid Album input, duplicate column names
252 EXPECT_EQ(iter, nullptr);
253 }
254