1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include <memory>
17 #include <string>
18
19 #include "common/common.h"
20 #include "minddata/dataset/core/client.h"
21 #include "minddata/dataset/engine/datasetops/source/celeba_op.h"
22 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
23 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
24 #include "minddata/dataset/util/status.h"
25 #include "gtest/gtest.h"
26 #include "utils/log_adapter.h"
27 #include "securec.h"
28
29 using namespace mindspore::dataset;
30 using mindspore::LogStream;
31 using mindspore::ExceptionType::NoExceptionType;
32 using mindspore::MsLogLevel::ERROR;
33
34 // std::shared_ptr<RepeatOp> Repeat(int repeat_cnt);
35
36 // std::shared_ptr<ExecutionTree> Build(std::vector<std::shared_ptr<DatasetOp>> ops);
37
Celeba(int32_t num_workers,int32_t queue_size,const std::string & dir,std::shared_ptr<SamplerRT> sampler=nullptr,bool decode=false,const std::string & dataset_type="all")38 std::shared_ptr<CelebAOp> Celeba(int32_t num_workers, int32_t queue_size, const std::string &dir,
39 std::shared_ptr<SamplerRT> sampler = nullptr, bool decode = false,
40 const std::string &dataset_type = "all") {
41 if (sampler == nullptr) {
42 const int64_t num_samples = 0;
43 const int64_t start_index = 0;
44 sampler = std::make_shared<SequentialSamplerRT>(start_index, num_samples);
45 }
46
47 std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
48 (void)schema->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1));
49 (void)schema->AddColumn(ColDescriptor("attr", DataType(DataType::DE_UINT32), TensorImpl::kFlexible, 1));
50
51 std::shared_ptr<ConfigManager> config_manager = GlobalContext::config_manager();
52 auto op_connector_size = config_manager->op_connector_size();
53 std::set<std::string> extensions = {};
54 std::shared_ptr<CelebAOp> so = std::make_shared<CelebAOp>(num_workers, dir, op_connector_size, decode, dataset_type,
55 extensions, std::move(schema), std::move(sampler));
56 return so;
57 }
58
59 class MindDataTestCelebaDataset : public UT::DatasetOpTesting {
60 protected:
61 };
62
TEST_F(MindDataTestCelebaDataset,TestSequentialCeleba)63 TEST_F(MindDataTestCelebaDataset, TestSequentialCeleba) {
64 std::string dir = datasets_root_path_ + "/testCelebAData/";
65 uint32_t expect_labels[4][40] = {{0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
66 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
67 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
68 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
69 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
70 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
71 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
72 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}};
73 uint32_t count = 0;
74 auto tree = Build({Celeba(16, 2, dir)});
75 tree->Prepare();
76 Status rc = tree->Launch();
77 if (rc.IsError()) {
78 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";
79 EXPECT_TRUE(false);
80 } else {
81 DatasetIterator di(tree);
82 TensorMap tensor_map;
83 ASSERT_OK(di.GetNextAsMap(&tensor_map));
84 EXPECT_TRUE(rc.IsOk());
85 while (tensor_map.size() != 0) {
86 uint32_t label;
87 for (int index = 0; index < 40; index++) {
88 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index});
89 EXPECT_TRUE(expect_labels[count][index] == label);
90 }
91 count++;
92 ASSERT_OK(di.GetNextAsMap(&tensor_map));
93 }
94 EXPECT_TRUE(count == 4);
95 }
96 }
97
TEST_F(MindDataTestCelebaDataset,TestCelebaRepeat)98 TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) {
99 std::string dir = datasets_root_path_ + "/testCelebAData/";
100 uint32_t expect_labels[8][40] = {{0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
101 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
102 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
103 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
104 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
105 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
106 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
107 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
108 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
109 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
110 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
111 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
112 {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
113 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1},
114 {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
115 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}};
116 uint32_t count = 0;
117 auto op1 = Celeba(16, 2, dir);
118 auto op2 = Repeat(2);
119 auto tree = Build({op1, op2});
120 op1->SetTotalRepeats(2);
121 op1->SetNumRepeatsPerEpoch(2);
122 tree->Prepare();
123 Status rc = tree->Launch();
124 if (rc.IsError()) {
125 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";
126 EXPECT_TRUE(false);
127 } else {
128 DatasetIterator di(tree);
129 TensorMap tensor_map;
130 ASSERT_OK(di.GetNextAsMap(&tensor_map));
131 EXPECT_TRUE(rc.IsOk());
132 while (tensor_map.size() != 0) {
133 uint32_t label;
134 for (int index = 0; index < 40; index++) {
135 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index});
136 EXPECT_TRUE(expect_labels[count][index] == label);
137 }
138 count++;
139 ASSERT_OK(di.GetNextAsMap(&tensor_map));
140 }
141 EXPECT_TRUE(count == 8);
142 }
143 }
144
TEST_F(MindDataTestCelebaDataset,TestSubsetRandomSamplerCeleba)145 TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) {
146 std::vector<int64_t> indices({1});
147 int64_t num_samples = 0;
148 std::shared_ptr<SamplerRT> sampler = std::make_shared<SubsetRandomSamplerRT>(indices, num_samples);
149 uint32_t expect_labels[1][40] = {{0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
150 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}};
151 std::string dir = datasets_root_path_ + "/testCelebAData/";
152 uint32_t count = 0;
153 auto tree = Build({Celeba(16, 2, dir, std::move(sampler))});
154 tree->Prepare();
155 Status rc = tree->Launch();
156 if (rc.IsError()) {
157 MS_LOG(ERROR) << "Return code error detected during tree launch: " << rc.ToString() << ".";
158 EXPECT_TRUE(false);
159 } else {
160 DatasetIterator di(tree);
161 TensorMap tensor_map;
162 ASSERT_OK(di.GetNextAsMap(&tensor_map));
163 EXPECT_TRUE(rc.IsOk());
164 while (tensor_map.size() != 0) {
165 uint32_t label;
166 for (int index = 0; index < 40; index++) {
167 tensor_map["attr"]->GetItemAt<uint32_t>(&label, {index});
168 EXPECT_TRUE(expect_labels[count][index] == label);
169 }
170 count++;
171 ASSERT_OK(di.GetNextAsMap(&tensor_map));
172 }
173 EXPECT_TRUE(count == 1);
174 }
175 }
176