1 /**
2 * Copyright 2020-2021 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "minddata/dataset/engine/tree_adapter.h"
18 #include "common/common.h"
19 #include "minddata/dataset/core/tensor_row.h"
20 #include "minddata/dataset/include/dataset/datasets.h"
21 #include "minddata/dataset/include/dataset/transforms.h"
22
23 // IR non-leaf nodes
24 #include "minddata/dataset/engine/ir/datasetops/batch_node.h"
25 #include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h"
26 #include "minddata/dataset/engine/ir/datasetops/concat_node.h"
27 #include "minddata/dataset/engine/ir/datasetops/map_node.h"
28 #include "minddata/dataset/engine/ir/datasetops/project_node.h"
29 #include "minddata/dataset/engine/ir/datasetops/rename_node.h"
30 #include "minddata/dataset/engine/ir/datasetops/shuffle_node.h"
31 #include "minddata/dataset/engine/ir/datasetops/skip_node.h"
32 #include "minddata/dataset/engine/ir/datasetops/zip_node.h"
33
34
35 using namespace mindspore::dataset;
36 using mindspore::dataset::Tensor;
37
38 class MindDataTestTreeAdapter : public UT::DatasetOpTesting {
39 protected:
40 };
41
TEST_F(MindDataTestTreeAdapter,TestSimpleTreeAdapter)42 TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) {
43 MS_LOG(INFO) << "Doing MindDataTestTreeAdapter-TestSimpleTreeAdapter.";
44
45 // Create a Mnist Dataset
46 std::string folder_path = datasets_root_path_ + "/testMnistData/";
47 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 4));
48 EXPECT_NE(ds, nullptr);
49
50 ds = ds->Batch(2);
51 EXPECT_NE(ds, nullptr);
52
53 mindspore::dataset::TreeAdapter tree_adapter;
54
55 Status rc = tree_adapter.Compile(ds->IRNode(), 1);
56
57 EXPECT_TRUE(rc.IsOk());
58
59 const std::unordered_map<std::string, int32_t> map = {{"label", 1}, {"image", 0}};
60 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map);
61
62 std::vector<size_t> row_sizes = {2, 2, 0};
63
64 TensorRow row;
65 for (size_t sz : row_sizes) {
66 rc = tree_adapter.GetNext(&row);
67 EXPECT_TRUE(rc.IsOk());
68 EXPECT_EQ(row.size(), sz);
69 }
70
71 rc = tree_adapter.GetNext(&row);
72 EXPECT_TRUE(rc.IsError());
73 const std::string err_msg = rc.ToString();
74 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos);
75 }
76
TEST_F(MindDataTestTreeAdapter,TestTreeAdapterWithRepeat)77 TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) {
78 MS_LOG(INFO) << "Doing MindDataTestTreeAdapter-TestTreeAdapterWithRepeat.";
79
80 // Create a Mnist Dataset
81 std::string folder_path = datasets_root_path_ + "/testMnistData/";
82 std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", std::make_shared<SequentialSampler>(0, 3));
83 EXPECT_NE(ds, nullptr);
84
85 ds = ds->Batch(2, false);
86 EXPECT_NE(ds, nullptr);
87
88 mindspore::dataset::TreeAdapter tree_adapter;
89
90 Status rc = tree_adapter.Compile(ds->IRNode(), 2);
91 EXPECT_TRUE(rc.IsOk());
92
93 const std::unordered_map<std::string, int32_t> map = tree_adapter.GetColumnNameMap();
94 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map);
95
96 std::vector<size_t> row_sizes = {2, 2, 0, 2, 2, 0};
97
98 TensorRow row;
99 for (size_t sz : row_sizes) {
100 rc = tree_adapter.GetNext(&row);
101 EXPECT_TRUE(rc.IsOk());
102 EXPECT_EQ(row.size(), sz);
103 }
104 rc = tree_adapter.GetNext(&row);
105 const std::string err_msg = rc.ToString();
106 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos);
107 }
108
TEST_F(MindDataTestTreeAdapter,TestProjectMapTreeAdapter)109 TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) {
110 MS_LOG(INFO) << "Doing MindDataTestPipeline-TestProjectMap.";
111
112 // Create an ImageFolder Dataset
113 std::string folder_path = datasets_root_path_ + "/testPK/data/";
114 std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<SequentialSampler>(0, 2));
115 EXPECT_NE(ds, nullptr);
116
117 // Create objects for the tensor ops
118 std::shared_ptr<TensorTransform> one_hot = std::make_shared<transforms::OneHot>(10);
119 EXPECT_NE(one_hot, nullptr);
120
121 // Create a Map operation, this will automatically add a project after map
122 ds = ds->Map({one_hot}, {"label"}, {"label"}, {"label"});
123 EXPECT_NE(ds, nullptr);
124
125 mindspore::dataset::TreeAdapter tree_adapter;
126
127 Status rc = tree_adapter.Compile(ds->IRNode(), 2);
128
129 EXPECT_TRUE(rc.IsOk());
130
131 const std::unordered_map<std::string, int32_t> map = {{"label", 0}};
132 EXPECT_EQ(tree_adapter.GetColumnNameMap(), map);
133
134 std::vector<size_t> row_sizes = {1, 1, 0, 1, 1, 0};
135 TensorRow row;
136
137 for (size_t sz : row_sizes) {
138 rc = tree_adapter.GetNext(&row);
139 EXPECT_TRUE(rc.IsOk());
140 EXPECT_EQ(row.size(), sz);
141 }
142 rc = tree_adapter.GetNext(&row);
143 const std::string err_msg = rc.ToString();
144 EXPECT_TRUE(err_msg.find("EOF buffer encountered.") != err_msg.npos);
145 }
146