1 /**
2 * Copyright 2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "common/common.h"
17 #include "minddata/dataset/include/dataset/datasets.h"
18
19 using namespace mindspore::dataset;
20 using mindspore::dataset::Tensor;
21
22 class MindDataTestEpochCtrl : public UT::DatasetOpTesting {
23 protected:
24 };
25
TEST_F(MindDataTestEpochCtrl,TestAutoInjectEpoch)26 TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
27 MS_LOG(INFO) << "Doing MindDataTestEpochCtrl-TestAutoInjectEpoch.";
28
29 int32_t img_class[4] = {0, 1, 2, 3};
30 int32_t num_epochs = 2 + std::rand() % 3;
31 int32_t sampler_size = 44;
32 int32_t class_size = 11;
33 MS_LOG(INFO) << "num_epochs: " << num_epochs;
34
35 // Create an ImageFolder Dataset
36 std::string folder_path = datasets_root_path_ + "/testPK/data/";
37 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, sampler_size));
38 ds = ds->SetNumWorkers(2);
39
40 // Create an iterator over the result of the above dataset
41 std::shared_ptr<Iterator> iter = ds->CreateIterator();
42 // Expect a valid iterator
43 ASSERT_NE(iter, nullptr);
44
45 uint64_t i = 0;
46 std::unordered_map<std::string, mindspore::MSTensor> row;
47
48 for (int epoch = 0; epoch < num_epochs; epoch++) {
49 // Iterate the dataset and get each row
50 ASSERT_OK(iter->GetNextRow(&row));
51
52 while (row.size() != 0) {
53 auto label = row["label"];
54 std::shared_ptr<Tensor> de_label;
55 int64_t label_value;
56 ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
57 ASSERT_OK(de_label->GetItemAt(&label_value, {}));
58 EXPECT_TRUE(img_class[(i % sampler_size) / class_size] == label_value);
59
60 ASSERT_OK(iter->GetNextRow(&row));
61 i++;
62 }
63 }
64
65 EXPECT_EQ(i, sampler_size * num_epochs);
66
67 // Try to fetch data beyond the specified number of epochs.
68 ASSERT_OK(iter->GetNextRow(&row));
69 EXPECT_EQ(row.size(), 2);
70
71 // Manually terminate the pipeline
72 iter->Stop();
73 }
74
TEST_F(MindDataTestEpochCtrl,TestEpoch)75 TEST_F(MindDataTestEpochCtrl, TestEpoch) {
76 MS_LOG(INFO) << "Doing MindDataTestEpochCtrl-TestEpoch.";
77
78 int32_t num_epochs = 1 + std::rand() % 4;
79 int32_t sampler_size = 7;
80 MS_LOG(INFO) << "num_epochs: " << num_epochs;
81
82 // Create an ImageFolder Dataset
83 std::string folder_path = datasets_root_path_ + "/testPK/data/";
84 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
85 ds = ds->SetNumWorkers(3);
86
87 // Create an iterator over the result of the above dataset
88 std::shared_ptr<Iterator> iter = ds->CreateIterator();
89 // Expect a valid iterator
90 ASSERT_NE(iter, nullptr);
91
92 // Iterate the dataset and get each row
93 uint64_t i = 0;
94 std::unordered_map<std::string, mindspore::MSTensor> row;
95
96 for (int epoch = 0; epoch < num_epochs; epoch++) {
97 ASSERT_OK(iter->GetNextRow(&row));
98 while (row.size() != 0) {
99 auto label = row["label"];
100 std::shared_ptr<Tensor> de_label;
101 int64_t label_value;
102 ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
103 ASSERT_OK(de_label->GetItemAt(&label_value, {}));
104 EXPECT_TRUE(label_value >= 0 && label_value <= 3);
105
106 ASSERT_OK(iter->GetNextRow(&row));
107 i++;
108 }
109 }
110
111 // Verify correct number of rows fetched
112 EXPECT_EQ(i, sampler_size * num_epochs);
113
114 // Try to fetch data beyond the specified number of epochs.
115 ASSERT_OK(iter->GetNextRow(&row));
116 EXPECT_EQ(row.size(), 2);
117
118 // Manually terminate the pipeline
119 iter->Stop();
120 }
121
TEST_F(MindDataTestEpochCtrl,TestRepeatEpoch)122 TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {
123 MS_LOG(INFO) << "Doing MindDataTestEpochCtrl-TestRepeatEpoch.";
124
125 int32_t num_epochs = 2 + std::rand() % 5;
126 int32_t num_repeats = 3;
127 int32_t sampler_size = 7;
128 MS_LOG(INFO) << "num_epochs: " << num_epochs;
129
130 // Create an ImageFolder Dataset
131 std::string folder_path = datasets_root_path_ + "/testPK/data/";
132 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(0, sampler_size));
133 ds = ds->SetNumWorkers(3);
134 ds = ds->Repeat(num_repeats);
135
136 // Create an iterator over the result of the above dataset
137 std::shared_ptr<Iterator> iter = ds->CreateIterator();
138 // Expect a valid iterator
139 ASSERT_NE(iter, nullptr);
140
141 // Iterate the dataset and get each row
142 uint64_t i = 0;
143 std::unordered_map<std::string, mindspore::MSTensor> row;
144
145 for (int epoch = 0; epoch < num_epochs; epoch++) {
146 ASSERT_OK(iter->GetNextRow(&row));
147 while (row.size() != 0) {
148 auto label = row["label"];
149 std::shared_ptr<Tensor> de_label;
150 int64_t label_value;
151 ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
152 ASSERT_OK(de_label->GetItemAt(&label_value, {}));
153 EXPECT_TRUE(label_value >= 0 && label_value <= 3);
154
155 ASSERT_OK(iter->GetNextRow(&row));
156 i++;
157 }
158 }
159
160 // Verify correct number of rows fetched
161 EXPECT_EQ(i, sampler_size * num_repeats * num_epochs);
162
163 // Try to fetch data beyond the specified number of epochs.
164 ASSERT_OK(iter->GetNextRow(&row));
165 EXPECT_EQ(row.size(), 2);
166
167 // Manually terminate the pipeline
168 iter->Stop();
169 }
170
TEST_F(MindDataTestEpochCtrl,TestRepeatRepeatEpoch)171 TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {
172 MS_LOG(INFO) << "Doing MindDataTestEpochCtrl-TestRepeatRepeatEpoch.";
173
174 int32_t num_epochs = 1 + std::rand() % 5;
175 int32_t num_repeats[2] = {2, 3};
176 int32_t sampler_size = 11;
177 MS_LOG(INFO) << "num_epochs: " << num_epochs;
178
179 // Create an ImageFolder Dataset
180 std::string folder_path = datasets_root_path_ + "/testPK/data/";
181 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(5, sampler_size));
182 ds = ds->Repeat(num_repeats[0]);
183 ds = ds->Repeat(num_repeats[1]);
184
185 // Create an iterator over the result of the above dataset
186 std::shared_ptr<Iterator> iter = ds->CreateIterator();
187 // Expect a valid iterator
188 ASSERT_NE(iter, nullptr);
189
190 // Iterate the dataset and get each row
191 uint64_t i = 0;
192 std::unordered_map<std::string, mindspore::MSTensor> row;
193
194 for (int epoch = 0; epoch < num_epochs; epoch++) {
195 ASSERT_OK(iter->GetNextRow(&row));
196 while (row.size() != 0) {
197 auto label = row["label"];
198 std::shared_ptr<Tensor> de_label;
199 int64_t label_value;
200 ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
201 ASSERT_OK(de_label->GetItemAt(&label_value, {}));
202 EXPECT_TRUE(label_value >= 0 && label_value <= 3);
203
204 ASSERT_OK(iter->GetNextRow(&row));
205 i++;
206 }
207 }
208
209 // Verify correct number of rows fetched
210 EXPECT_EQ(i, sampler_size * num_repeats[0] * num_repeats[1] * num_epochs);
211
212 // Try to fetch data beyond the specified number of epochs.
213 ASSERT_OK(iter->GetNextRow(&row));
214 EXPECT_EQ(row.size(), 2);
215
216 // Manually terminate the pipeline
217 iter->Stop();
218 }
219